You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/04/13 21:18:14 UTC
[07/33] incubator-usergrid git commit: New mapping started.
New mapping started.
Basic object -> field mapping tested
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1e3ea744
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1e3ea744
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1e3ea744
Branch: refs/heads/two-dot-o-dev
Commit: 1e3ea744e2bffb73d4b58889306a5c9c65c6be1c
Parents: fb81d64
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Apr 2 20:18:59 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Apr 2 20:18:59 2015 -0600
----------------------------------------------------------------------
.../persistence/model/entity/EntityMap.java | 16 +-
.../model/entity/EntityToMapConverter.java | 9 +-
.../persistence/model/field/ListField.java | 6 +
.../persistence/index/impl/DeIndexRequest.java | 4 +-
.../persistence/index/impl/EntityField.java | 135 +++++++
.../index/impl/EntityMappingParser.java | 246 +++++++++++++
.../index/impl/EntityToMapConverter.java | 193 ++++------
.../impl/EsApplicationEntityIndexImpl.java | 11 +-
.../index/impl/EsEntityIndexImpl.java | 2 +-
.../persistence/index/impl/EsQueryVistor.java | 29 +-
.../persistence/index/impl/FieldParser.java | 45 +++
.../persistence/index/impl/IndexRequest.java | 10 +-
.../persistence/index/impl/IndexingUtils.java | 165 +++++----
.../impl/SearchRequestBuilderStrategy.java | 85 +++--
.../index/migration/LegacyIndexIdentifier.java | 2 +-
.../persistence/index/usergrid-mappings.json | 67 +++-
.../persistence/index/impl/EntityIndexTest.java | 8 +-
.../index/impl/EntityToMapConverterTest.java | 351 +++++++++++++++++++
18 files changed, 1083 insertions(+), 301 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityMap.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityMap.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityMap.java
index 5dbd1fb..fa4ad06 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityMap.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityMap.java
@@ -25,7 +25,7 @@ public class EntityMap extends HashMap<String,Object> {
public EntityMap(Id id,UUID version){
super();
setId(id);
- setVersion(version);
+ setVersion( version );
}
@JsonIgnore
@@ -58,6 +58,20 @@ public class EntityMap extends HashMap<String,Object> {
return entityToMapConverter.toMap(entity);
}
+
+ /**
+ * Return true if the value is a location field
+ * @param fieldValue
+ * @return
+ */
+ public static boolean isLocationField(Map<String, ?> fieldValue){
+ if(fieldValue.size() != 2){
+ return false;
+ }
+
+ return fieldValue.containsKey( EntityToMapConverter.LAT ) && fieldValue.containsKey( EntityToMapConverter.LON );
+ }
+
public void clearFields() {
this.remove(ID_KEY);
this.remove(TYPE_KEY);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityToMapConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityToMapConverter.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityToMapConverter.java
index 6b36af7..3865e62 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityToMapConverter.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityToMapConverter.java
@@ -29,6 +29,9 @@ import java.util.*;
* abstract conversion to Map<String,Object> form EntityObject
*/
public class EntityToMapConverter{
+ public static final String LAT = "lat";
+ public static final String LON = "lon";
+
public static ObjectMapper objectMapper = new ObjectMapper( );
/**
* Convert Entity to Map, adding version_ug_field and a {name}_ug_analyzed field for each
@@ -74,9 +77,9 @@ public class EntityToMapConverter{
Map<String, Object> locMap = new HashMap<String, Object>();
// field names lat and lon trigger ElasticSearch geo location
- locMap.put("lat", locField.getValue().getLatitude());
- locMap.put("lon", locField.getValue().getLongitude());
- entityMap.put( field.getName(), field.getValue());
+ locMap.put( LAT, locField.getValue().getLatitude());
+ locMap.put( LON, locField.getValue().getLongitude());
+ entityMap.put( field.getName(), locMap);
} else if (f instanceof ByteArrayField) {
ByteArrayField bf = ( ByteArrayField ) f;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/ListField.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/ListField.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/ListField.java
index 6675d73..eb0f72f 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/ListField.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/ListField.java
@@ -55,4 +55,10 @@ public abstract class ListField<T> extends AbstractField<List<T>> {
return FieldTypeName.LIST;
}
+
+ public int size(){
+ return this.value.size();
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
index 591e060..b1fec73 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
@@ -24,6 +24,7 @@ import java.util.Arrays;
import java.util.UUID;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexEdge;
import org.apache.usergrid.persistence.index.SearchEdge;
import org.apache.usergrid.persistence.index.SearchType;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -54,10 +55,9 @@ public class DeIndexRequest implements BatchRequest {
public DeIndexRequest(String[] indexes, ApplicationScope applicationScope, SearchEdge searchEdge, Id id, UUID version) {
- String context = createContextName(applicationScope, searchEdge );
this.indexes = indexes;
this.entityTypes = SearchType.fromId(id).getTypeNames(applicationScope);
- this.documentId = createIndexDocId(applicationScope, id, version,context);
+ this.documentId = createIndexDocId(applicationScope, id, version,searchEdge);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityField.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityField.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityField.java
new file mode 100644
index 0000000..0585d6a
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityField.java
@@ -0,0 +1,135 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.util.HashMap;
+import java.util.Map;
+
+
+/**
+ * An object that represents an entity field.
+ *
+ * All factory methods should be used to create an instance of the field
+ */
+public class EntityField extends HashMap<String, Object> {
+
+
+ /**
+ * We only ever have 2 values in our field
+ */
+ private EntityField(){
+ super(2);
+ }
+
+
+ /**
+ * Set a string value into the field
+ */
+ public static EntityField create( final String fieldName, final String value ) {
+ EntityField field = new EntityField();
+ field.put( IndexingUtils.FIELD_NAME, fieldName );
+ field.put( IndexingUtils.FIELD_STRING, value );
+
+ return field;
+ }
+
+
+ /**
+ * Set a string value into the field
+ */
+ public static EntityField create( final String fieldName, final boolean value ) {
+ EntityField field = new EntityField();
+ field.put( IndexingUtils.FIELD_NAME, fieldName );
+ field.put( IndexingUtils.FIELD_BOOLEAN, value );
+
+ return field;
+ }
+
+
+
+
+ /**
+ * Set a string value into the field
+ */
+ public static EntityField create( final String fieldName, final int value ) {
+ EntityField field = new EntityField();
+ field.put( IndexingUtils.FIELD_NAME, fieldName );
+ field.put( IndexingUtils.FIELD_INT, value );
+
+ return field;
+ }
+
+
+ /**
+ * Set a string value into the field
+ */
+ public static EntityField create( final String fieldName, final long value ) {
+ EntityField field = new EntityField();
+ field.put( IndexingUtils.FIELD_NAME, fieldName );
+ field.put( IndexingUtils.FIELD_LONG, value );
+
+ return field;
+ }
+
+
+ /**
+ * Set a string value into the field
+ */
+ public static EntityField create( final String fieldName, final float value ) {
+ EntityField field = new EntityField();
+ field.put( IndexingUtils.FIELD_NAME, fieldName );
+ field.put( IndexingUtils.FIELD_FLOAT, value );
+
+ return field;
+ }
+
+
+ /**
+ * Set a string value into the field
+ */
+ public static EntityField create( final String fieldName, final double value ) {
+ EntityField field = new EntityField();
+ field.put( IndexingUtils.FIELD_NAME, fieldName );
+ field.put( IndexingUtils.FIELD_DOUBLE, value );
+
+ return field;
+ }
+
+
+
+
+ /**
+ * Set a location into our field
+ */
+ public static EntityField create( final String fieldName, final Map location) {
+ EntityField field = new EntityField();
+ field.put( IndexingUtils.FIELD_NAME, fieldName );
+ field.put( IndexingUtils.FIELD_LOCATION, location );
+
+ return field;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityMappingParser.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityMappingParser.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityMappingParser.java
new file mode 100644
index 0000000..406d879
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityMappingParser.java
@@ -0,0 +1,246 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+
+import org.apache.usergrid.persistence.model.entity.EntityMap;
+import org.apache.usergrid.persistence.model.field.value.Location;
+
+
+/**
+ * Our parser that will parse our entity map data, and return a collection of all field objects
+ *
+ * TODO: Decide if we're really getting rid of the Entity field object. If not, this can be much faster using a visitor
+ * pattern on the Entity
+ */
+public class EntityMappingParser implements FieldParser {
+
+
+ /**
+ * Our stack for fields
+ */
+ private Stack<String> fieldStack = new Stack();
+
+ private List<EntityField> fields = new ArrayList<>();
+
+
+ private void visit( final String value ) {
+ fields.add( EntityField.create( fieldStack.peek(), value ) );
+ }
+
+
+ private void visit( final boolean value ) {
+ fields.add( EntityField.create( fieldStack.peek(), value ) );
+ }
+
+
+
+ private void visit( final int value ) {
+ fields.add( EntityField.create( fieldStack.peek(), value ) );
+ }
+
+
+ private void visit( final long value ) {
+ fields.add( EntityField.create( fieldStack.peek(), value ) );
+ }
+
+
+ private void visit( final double value ) {
+ fields.add( EntityField.create( fieldStack.peek(), value ) );
+ }
+
+
+ private void visit( final float value ) {
+ fields.add( EntityField.create( fieldStack.peek(), value ) );
+ }
+
+
+
+
+ private void visit( final Map<String, ?> value ) {
+ //it's a location field, index it as such
+ if ( EntityMap.isLocationField( value ) ) {
+// fields.add( EntityField.create( fieldStack.peek(), value ) );
+ return;
+ }
+
+ iterate( value );
+ }
+
+
+ /**
+ * Iterate over a collection
+ */
+ private void iterate( final Collection value ) {
+
+ //no op
+ if ( value.size() == 0 ) {
+ return;
+ }
+
+ //fisit all the object element
+ for ( final Object element : value ) {
+ visitValue( element );
+ }
+ }
+
+
+ /**
+ * visit a value
+ */
+ private void visitValue( final Object value ) {
+
+ if ( isMap( value ) ) {
+ //if it's a location, then create a location field.
+ if ( EntityMap.isLocationField( (Map)value ) ) {
+ fields.add( EntityField.create( fieldStack.peek(), ( Map ) value ) );
+ return;
+ }
+
+ iterate( ( Map<String, ?> ) value );
+ }
+
+ //TODO figure out our nested array structure
+ else if ( isCollection( value ) ) {
+ iterate( ( Collection ) value );
+ }
+ else {
+ visitPrimitive( value );
+ }
+ }
+
+
+ /**
+ * Construct the correct primitive
+ */
+ private void visitPrimitive( final Object object ) {
+ if ( object instanceof String ) {
+ visit( ( String ) object );
+ return;
+ }
+
+ if ( object instanceof Boolean ) {
+ visit( ( Boolean ) object );
+ return;
+ }
+
+ if ( object instanceof Integer ) {
+ visit( ( Integer ) object );
+ return;
+ }
+
+ if ( object instanceof Long ) {
+ visit( ( Long ) object );
+ return;
+ }
+
+ if ( object instanceof Float ) {
+ visit( ( Float ) object );
+ return;
+ }
+
+ if ( object instanceof Double ) {
+ visit( ( Double ) object );
+ return;
+ }
+
+
+ }
+
+
+
+
+
+ private void iterate( final Map<String, ?> map ) {
+
+ for ( final Map.Entry<String, ?> jsonField : map.entrySet() ) {
+ pushField( jsonField.getKey() );
+ visitValue( jsonField.getValue() );
+ popField();
+ }
+ }
+
+
+ /**
+ * Return true if it's a map
+ */
+ private boolean isMap( final Object value ) {
+ return value instanceof Map;
+ }
+
+
+ /**
+ * Return true if it's a collection
+ */
+ private boolean isCollection( final Object value ) {
+ return value instanceof Collection;
+ }
+
+
+ /**
+ * Return true if this is a primitive (inverse of isMap and isCollection)
+ */
+ private boolean isPrimitive( final Object value ) {
+ return !isMap( value ) && !isCollection( value );
+ }
+
+
+ /**
+ * Push a new fieldname on to the stack
+ */
+ private void pushField( final String fieldName ) {
+ if ( fieldStack.isEmpty() ) {
+ fieldStack.push( fieldName );
+ return;
+ }
+
+
+ final String newFieldName = fieldStack.peek() + "." + fieldName;
+ fieldStack.push( newFieldName );
+ }
+
+
+ /**
+ * Pop a field name off the stack
+ */
+ private void popField() {
+ fieldStack.pop();
+ }
+
+
+ /**
+ * Parse the map field
+ */
+ public List<EntityField> parse( final Map<String, ?> map ) {
+ iterate( map );
+
+ return fields;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
index 04034cb..ff81672 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
@@ -16,152 +16,93 @@
*/
package org.apache.usergrid.persistence.index.impl;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.field.*;
-import org.apache.usergrid.persistence.model.field.value.EntityObject;
-import java.util.*;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexEdge;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.EntityMap;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.APPLICATION_ID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.EDGE_NAME_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.EDGE_NODE_ID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.EDGE_NODE_TYPE_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.EDGE_SEARCH_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.EDGE_TIMESTAMP_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_FIELDS;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_ID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_TYPE_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_VERSION_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.getType;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.idString;
/**
- * Classy class class.
+ * Convert a CP entity to an elasticsearch document
*/
public class EntityToMapConverter {
/**
* Set the entity as a map with the context
*
+ * @param applicationScope
* @param entity The entity
- * @param context The context this entity appears in
+ * @param indexEdge The edge this entity is indexed on
*/
- public static Map convert(ApplicationScope applicationScope, final Entity entity, final String context ) {
- final Map entityMap = entityToMap( entity );
+ public static Map<String, Object> convert(ApplicationScope applicationScope, final IndexEdge indexEdge, final Entity entity) {
+
+
+
+ final Map<String, Object> outputEntity = new HashMap<>();
+
+
+ final Id entityId = entity.getId();
+
+ /***
+ * Add our static fields for easier admin/debugging/reporting
+ ****/
+
+ outputEntity.put( ENTITY_ID_FIELDNAME, idString(entityId) );
+
+ outputEntity.put( ENTITY_VERSION_FIELDNAME, entity.getVersion() );
+
+ outputEntity.put( ENTITY_TYPE_FIELDNAME, getType( applicationScope, entityId ) );
+
+
+ outputEntity.put( APPLICATION_ID_FIELDNAME, idString( applicationScope.getApplication() ) );
+
+ outputEntity.put( EDGE_NODE_ID_FIELDNAME, idString( indexEdge.getNodeId() ) );
+
+ outputEntity.put( EDGE_NODE_TYPE_FIELDNAME, indexEdge.getNodeType() );
+
+ outputEntity.put( EDGE_NAME_FIELDNAME, indexEdge.getEdgeName() );
+
+ outputEntity.put( EDGE_TIMESTAMP_FIELDNAME, indexEdge.getTimestamp() );
+
//add the context for filtering later
- entityMap.put( ENTITY_CONTEXT_FIELDNAME, context );
+ outputEntity.put( EDGE_SEARCH_FIELDNAME, IndexingUtils.createContextName( applicationScope, indexEdge ) );
- //but the fieldname we have to prefix because we use query equality to seek this later.
- // TODO see if we can make this more declarative
- entityMap.put( ENTITYID_ID_FIELDNAME, IndexingUtils.idString(entity.getId()).toLowerCase() );
- entityMap.put( APPLICATION_ID_FIELDNAME, idString(applicationScope.getApplication()) );
+ //migrate the entity to map since we're ultimately going to use maps once we get rid of the Field objects
+ final EntityMap entityMap = EntityMap.fromEntity( entity );
+ entityMap.clearFields();
- return entityMap;
- }
+ //now visit our entity
+ final FieldParser parser = new EntityMappingParser();
- /**
- * Convert Entity to Map and Adding prefixes for types:
- * <pre>
- * su_ - String unanalyzed field
- * sa_ - String analyzed field
- * go_ - Location field nu_ - Number field
- * bu_ - Boolean field
- * </pre>
- */
- private static Map entityToMap( EntityObject entity ) {
-
- Map<String, Object> entityMap = new HashMap<String, Object>();
-
- for ( Object f : entity.getFields().toArray() ) {
-
- Field field = ( Field ) f;
-
- if ( f instanceof ListField) {
- List list = ( List ) field.getValue();
- entityMap.put(field.getName().toLowerCase(),
- new ArrayList( processCollectionForMap( list ) ) );
-
- if ( !list.isEmpty() ) {
- if ( list.get( 0 ) instanceof String ) {
- entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
- new ArrayList( processCollectionForMap( list ) ) );
- }
- }
- }
- else if ( f instanceof SetField) {
- Set set = ( Set ) field.getValue();
- entityMap.put( field.getName().toLowerCase(),
- new ArrayList( processCollectionForMap( set ) ) );
- }
- else if ( f instanceof EntityObjectField) {
- EntityObject eo = ( EntityObject ) field.getValue();
- entityMap.put( field.getName().toLowerCase(), entityToMap(eo) ); // recursion
- }
- else if ( f instanceof StringField ) {
-
- // index in lower case because Usergrid queries are case insensitive
- entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
- ( ( String ) field.getValue() ).toLowerCase() );
- entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
- ( ( String ) field.getValue() ).toLowerCase() );
- }
- else if ( f instanceof LocationField ) {
- LocationField locField = ( LocationField ) f;
- Map<String, Object> locMap = new HashMap<String, Object>();
-
- // field names lat and lon trigger ElasticSearch geo location
- locMap.put( "lat", locField.getValue().getLatitude() );
- locMap.put( "lon", locField.getValue().getLongitude() );
- entityMap.put( GEO_PREFIX + field.getName().toLowerCase(), locMap );
- }
- else if( f instanceof DoubleField || f instanceof FloatField){
- entityMap.put( DOUBLE_PREFIX + field.getName().toLowerCase(), field.getValue() );
- }
- else if( f instanceof LongField || f instanceof IntegerField){
- entityMap.put( LONG_PREFIX + field.getName().toLowerCase(), field.getValue() );
- }
- else if ( f instanceof BooleanField ) {
-
- entityMap.put( BOOLEAN_PREFIX + field.getName().toLowerCase(), field.getValue() );
- }
- else if ( f instanceof UUIDField ) {
-
- entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
- field.getValue().toString().toLowerCase() );
- }
- else {
- entityMap.put( field.getName().toLowerCase(), field.getValue() );
- }
- }
-
- return entityMap;
- }
+ final List<EntityField> fieldsToIndex = parser.parse( entityMap );
- private static Collection processCollectionForMap( final Collection c ) {
- if ( c.isEmpty() ) {
- return c;
- }
- List processed = new ArrayList();
- Object sample = c.iterator().next();
-
- if ( sample instanceof Entity ) {
- for ( Object o : c.toArray() ) {
- Entity e = ( Entity ) o;
- processed.add( entityToMap( e ) );
- }
- }
- else if ( sample instanceof List ) {
- for ( Object o : c.toArray() ) {
- List list = ( List ) o;
- processed.add( processCollectionForMap( list ) ); // recursion;
- }
- }
- else if ( sample instanceof Set ) {
- for ( Object o : c.toArray() ) {
- Set set = ( Set ) o;
- processed.add( processCollectionForMap( set ) ); // recursion;
- }
- }
- else {
- for ( Object o : c.toArray() ) {
- processed.add( o );
- }
- }
- return processed;
+ //add our fields
+ outputEntity.put( ENTITY_FIELDS, fieldsToIndex );
+
+
+ return outputEntity;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 171f1c7..d76dab7 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -72,7 +72,7 @@ import com.google.inject.assistedinject.Assisted;
import rx.Observable;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.APPLICATION_ID_FIELDNAME;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.parseIndexDocId;
/**
@@ -306,14 +306,9 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
for ( SearchHit hit : hits ) {
- String[] idparts = hit.getId().split( SPLITTER );
- String id = idparts[0];
- String type = idparts[1];
- String version = idparts[2];
+ final CandidateResult candidateResult = parseIndexDocId( hit.getId() );
- Id entityId = new SimpleId( UUID.fromString( id ), type );
-
- candidates.add( new CandidateResult( entityId, UUID.fromString( version ) ) );
+ candidates.add( candidateResult );
}
final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 09bc4ff..7110a22 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -88,7 +88,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
private static final String VERIFY_TYPE = "entity";
private static final ImmutableMap<String, Object> DEFAULT_PAYLOAD =
- ImmutableMap.<String, Object>builder().put(IndexingUtils.ENTITYID_ID_FIELDNAME, UUIDGenerator.newTimeUUID().toString()).build();
+ ImmutableMap.<String, Object>builder().put(IndexingUtils.ENTITY_ID_FIELDNAME, UUIDGenerator.newTimeUUID().toString()).build();
private static final MatchAllQueryBuilder MATCH_ALL_QUERY_BUILDER = QueryBuilders.matchAllQuery();
private final FailureMonitorImpl.IndexIdentifier indexIdentifier;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
index 8e8b43c..39b101b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
@@ -175,9 +175,7 @@ public class EsQueryVistor implements QueryVisitor {
float lon = op.getLongitude().getFloatValue();
float distance = op.getDistance().getFloatValue();
- if ( !name.startsWith( GEO_PREFIX )) {
- name = GEO_PREFIX + name;
- }
+
FilterBuilder fb = FilterBuilders.geoDistanceFilter( name )
.lat( lat ).lon( lon ).distance( distance, DistanceUnit.METERS );
@@ -310,41 +308,26 @@ public class EsQueryVistor implements QueryVisitor {
private String addAnalyzedStringPrefix( String name ) {
- if ( name.startsWith( ANALYZED_STRING_PREFIX ) ) {
- return name;
- }
- return ANALYZED_STRING_PREFIX + name;
+ return name;
}
private String addStringPrefix( String name ) {
- if ( name.startsWith( STRING_PREFIX ) ) {
- return name;
- }
- return STRING_PREFIX + name;
+ return name;
}
private String addDoublePrefix( String name ) {
- if ( name.startsWith( DOUBLE_PREFIX ) ) {
- return name;
- }
- return DOUBLE_PREFIX + name;
+ return name;
}
private String addLongPrefix( String name ) {
- if ( name.startsWith( LONG_PREFIX ) ) {
- return name;
- }
- return LONG_PREFIX + name;
+ return name;
}
private String addBooleanPrefix( String name ) {
- if ( name.startsWith( BOOLEAN_PREFIX ) ) {
- return name;
- }
- return BOOLEAN_PREFIX + name;
+ return name;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FieldParser.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FieldParser.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FieldParser.java
new file mode 100644
index 0000000..663d2a5
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FieldParser.java
@@ -0,0 +1,45 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * Interface for visiting a Map as JSON
+ */
+public interface FieldParser {
+
+
+ /**
+ * Visit an object. Start the root object here.
+ * @param map
+ */
+ List<EntityField> parse( final Map<String, ?> map );
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
index 37ed590..f82bc16 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
@@ -23,6 +23,7 @@ package org.apache.usergrid.persistence.index.impl;
import java.util.Map;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexEdge;
import org.apache.usergrid.persistence.index.SearchEdge;
import org.apache.usergrid.persistence.index.SearchType;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -49,15 +50,12 @@ public class IndexRequest implements BatchRequest {
public Map<String, Object> data;
- public IndexRequest( final String writeAlias, final ApplicationScope applicationScope, SearchEdge searchEdge, Entity entity) {
- this(writeAlias, applicationScope, createContextName(applicationScope, searchEdge ), entity);
- }
+ public IndexRequest( final String writeAlias, final ApplicationScope applicationScope, IndexEdge indexEdge, Entity entity) {
+ this(writeAlias, applicationScope, SearchType.fromId( entity.getId() ),IndexingUtils.createIndexDocId(applicationScope, entity,indexEdge), EntityToMapConverter.convert(applicationScope,indexEdge, entity));
- public IndexRequest( final String writeAlias, final ApplicationScope applicationScope, String context , Entity entity) {
- this(writeAlias, applicationScope, SearchType.fromId(entity.getId()),IndexingUtils.createIndexDocId(applicationScope, entity,context), EntityToMapConverter.convert(applicationScope,entity, context));
}
- public IndexRequest( final String writeAlias, final ApplicationScope applicationScope,SearchType searchType, String documentId, Map<String, Object> data) {
+ public IndexRequest( final String writeAlias, final ApplicationScope applicationScope, SearchType searchType, String documentId, Map<String, Object> data) {
this.writeAlias = writeAlias;
this.entityType = searchType.getTypeName(applicationScope);
this.data = data;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 6fd1f33..d1a030f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -18,125 +18,160 @@ package org.apache.usergrid.persistence.index.impl;/*
*/
-import java.io.IOException;
import java.util.UUID;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.CandidateResult;
+import org.apache.usergrid.persistence.index.IndexEdge;
import org.apache.usergrid.persistence.index.SearchEdge;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-
-import org.apache.commons.lang.NotImplementedException;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
public class IndexingUtils {
- public static final String STRING_PREFIX = "su_";
- public static final String ANALYZED_STRING_PREFIX = "sa_";
- public static final String GEO_PREFIX = "go_";
- public static final String LONG_PREFIX = "long_";
- public static final String DOUBLE_PREFIX = "long_";
- public static final String BOOLEAN_PREFIX = "bu_";
- public static final String ARRAY_PREFIX = "ar_";
- public static final String SET_PREFIX = "set_";
- public static final String EO_PREFIX = "eo_";
+ // These are not allowed in document type names: _ . , | #
+ public static final String FIELD_SEPERATOR = "__";
- public static final String SPLITTER = "\\__";
+ public static final String ID_SEPERATOR = "_";
- // These are not allowed in document type names: _ . , | #
- public static final String SEPARATOR = "__";
+ /**
+ * Reserved UG fields in the document
+ */
+ public static final String APPLICATION_ID_FIELDNAME = "applicationId";
+
+ public static final String ENTITY_ID_FIELDNAME = "entityId";
+
+ public static final String ENTITY_VERSION_FIELDNAME = "entityVersion";
+
+ public static final String ENTITY_TYPE_FIELDNAME = "entityType";
+
+ public static final String EDGE_NODE_ID_FIELDNAME = "edgeNodeId";
- //
- // Reserved UG fields.
- //
- public static final String APPLICATION_ID_FIELDNAME = "ug_applicationId";
+ public static final String EDGE_NAME_FIELDNAME = "edgeName";
- public static final String ENTITY_CONTEXT_FIELDNAME = "ug_context";
+ public static final String EDGE_NODE_TYPE_FIELDNAME = "edgeType";
- public static final String ENTITYID_ID_FIELDNAME = "ug_entityId";
+ public static final String EDGE_TIMESTAMP_FIELDNAME = "edgeTimestamp";
- public static final String ENTITY_VERSION_FIELDNAME = "ug_entityVersion";
+ public static final String EDGE_SEARCH_FIELDNAME = "edgeSearch";
+
+ public static final String ENTITY_FIELDS = "fields";
- public static final String DOC_VALUES_KEY = "doc_values";
/**
- * Create our sub scope. This is the ownerUUID + type
- * @param scope
- * @return
- */
- public static String createContextName(final ApplicationScope applicationScope, final SearchEdge scope ) {
- StringBuilder sb = new StringBuilder();
- idString(sb,applicationScope.getApplication());
- sb.append(SEPARATOR);
- idString(sb, scope.getNodeId());
- sb.append( SEPARATOR );
- sb.append( scope.getEdgeName() );
- return sb.toString();
- }
+ * Reserved field types in our document
+ */
+ public static final String FIELD_NAME = "name";
+ public static final String FIELD_BOOLEAN = "boolean";
+ public static final String FIELD_INT = "int";
+ public static final String FIELD_LONG = "long";
+ public static final String FIELD_FLOAT = "float";
+ public static final String FIELD_DOUBLE = "double";
+ public static final String FIELD_LOCATION = "location";
+ public static final String FIELD_STRING = "string";
+
+ //The value appended to the string field when it's an exact match. Should only be used on search, never index
+ public static final String FIELD_STRING_EQUALS = FIELD_STRING + ".exact";
+
+
+ /**
+ * Create our sub scope. This is the ownerUUID + type
+ */
+ public static String createContextName( final ApplicationScope applicationScope, final SearchEdge scope ) {
+ StringBuilder sb = new StringBuilder();
+ idString( sb, applicationScope.getApplication() );
+ sb.append( FIELD_SEPERATOR );
+ idString( sb, scope.getNodeId() );
+ sb.append( FIELD_SEPERATOR );
+ sb.append( scope.getEdgeName() );
+ sb.append( FIELD_SEPERATOR );
+ sb.append( scope.getNodeType() );
+ return sb.toString();
+ }
/**
* Append the id to the string
- * @param builder
- * @param id
*/
- public static final void idString(final StringBuilder builder, final Id id){
- builder.append( id.getUuid() ).append( SEPARATOR )
- .append(id.getType());
+ public static final void idString( final StringBuilder builder, final Id id ) {
+ builder.append( id.getUuid() ).append( ID_SEPERATOR ).append( id.getType().toLowerCase() );
}
/**
* Turn the id into a string
- * @param id
- * @return
*/
- public static final String idString(final Id id){
- final StringBuilder sb = new StringBuilder( );
- idString(sb, id);
+ public static final String idString( final Id id ) {
+ final StringBuilder sb = new StringBuilder();
+ idString( sb, id );
return sb.toString();
}
+
/**
* Create the index doc from the given entity
- * @param entity
- * @return
*/
- public static String createIndexDocId(final ApplicationScope applicationScope, final Entity entity, final String context) {
- return createIndexDocId(applicationScope, entity.getId(), entity.getVersion(), context);
+ public static String createIndexDocId( final ApplicationScope applicationScope, final Entity entity,
+ final IndexEdge indexEdge ) {
+ return createIndexDocId( applicationScope, entity.getId(), entity.getVersion(), indexEdge );
}
/**
* Create the doc Id. This is the entitie's type + uuid + version
- * @param entityId
- * @param version
- * @para context The context it's indexed in
- * @return
*/
- public static String createIndexDocId( final ApplicationScope applicationScope, final Id entityId, final UUID version, final String context) {
+ public static String createIndexDocId( final ApplicationScope applicationScope, final Id entityId,
+ final UUID version, final SearchEdge searchEdge ) {
StringBuilder sb = new StringBuilder();
idString( applicationScope.getApplication() );
- sb.append( SEPARATOR );
- idString(sb, entityId);
- sb.append( SEPARATOR );
- sb.append( version.toString() ).append( SEPARATOR );
- sb.append( context);
+ sb.append( FIELD_SEPERATOR );
+ idString( sb, entityId );
+ sb.append( FIELD_SEPERATOR );
+ sb.append( version.toString() );
+
+ sb.append( FIELD_SEPERATOR );
+
+ idString( searchEdge.getNodeId() );
+
+ sb.append( FIELD_SEPERATOR );
+ sb.append( searchEdge.getEdgeName() );
+ sb.append( FIELD_SEPERATOR );
+ sb.append( searchEdge.getNodeType() );
+
return sb.toString();
}
+ /**
+ * Parse the document id into a candidate result
+ * @param documentId
+ * @return
+ */
+ public static CandidateResult parseIndexDocId( final String documentId ) {
+
+ String[] idparts = documentId.split( FIELD_SEPERATOR );
+ String entityIdString = idparts[1];
+ String version = idparts[2];
+
+ final String[] entityIdParts = entityIdString.split( ID_SEPERATOR );
+ Id entityId = new SimpleId( UUID.fromString( entityIdParts[0] ), entityIdParts[1] );
- public static String getType(ApplicationScope applicationScope, Id entityId) {
- return getType(applicationScope,entityId.getType());
+ return new CandidateResult( entityId, UUID.fromString( version ) );
}
- public static String getType(ApplicationScope applicationScope, String type) {
- return idString(applicationScope.getApplication()) + SEPARATOR + type;
+
+ public static String getType( ApplicationScope applicationScope, Id entityId ) {
+ return getType( applicationScope, entityId.getType() );
+ }
+
+
+ public static String getType( ApplicationScope applicationScope, String type ) {
+ return idString( applicationScope.getApplication() ) + FIELD_SEPERATOR + type;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
index 8d4d599..18d4228 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
@@ -42,10 +42,6 @@ import org.apache.usergrid.persistence.index.query.tree.QueryVisitor;
import com.google.common.base.Preconditions;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.DOUBLE_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.LONG_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
/**
@@ -90,45 +86,46 @@ public class SearchRequestBuilderStrategy {
srb = srb.setFrom(0).setSize(limit);
for (SortPredicate sp : query.getSortPredicates()) {
-
- final SortOrder order;
- if (sp.getDirection().equals( SortPredicate.SortDirection.ASCENDING)) {
- order = SortOrder.ASC;
- } else {
- order = SortOrder.DESC;
- }
-
- // we do not know the type of the "order by" property and so we do not know what
- // type prefix to use. So, here we add an order by clause for every possible type
- // that you can order by: string, number and boolean and we ask ElasticSearch
- // to ignore any fields that are not present.
-
- final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
- final FieldSortBuilder stringSort = SortBuilders.fieldSort(stringFieldName)
- .order(order).ignoreUnmapped(true);
- srb.addSort(stringSort);
-
- logger.debug(" Sort: {} order by {}", stringFieldName, order.toString());
-
- final String longFieldName = LONG_PREFIX + sp.getPropertyName();
- final FieldSortBuilder longSort = SortBuilders.fieldSort(longFieldName)
- .order(order).ignoreUnmapped(true);
- srb.addSort(longSort);
- logger.debug(" Sort: {} order by {}", longFieldName, order.toString());
-
-
- final String doubleFieldName = DOUBLE_PREFIX + sp.getPropertyName();
- final FieldSortBuilder doubleSort = SortBuilders.fieldSort(doubleFieldName)
- .order(order).ignoreUnmapped(true);
- srb.addSort(doubleSort);
- logger.debug(" Sort: {} order by {}", doubleFieldName, order.toString());
-
-
- final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
- final FieldSortBuilder booleanSort = SortBuilders.fieldSort(booleanFieldName)
- .order(order).ignoreUnmapped(true);
- srb.addSort(booleanSort);
- logger.debug(" Sort: {} order by {}", booleanFieldName, order.toString());
+ throw new RuntimeException( "Fix me" );
+
+// final SortOrder order;
+// if (sp.getDirection().equals( SortPredicate.SortDirection.ASCENDING)) {
+// order = SortOrder.ASC;
+// } else {
+// order = SortOrder.DESC;
+// }
+//
+// // we do not know the type of the "order by" property and so we do not know what
+// // type prefix to use. So, here we add an order by clause for every possible type
+// // that you can order by: string, number and boolean and we ask ElasticSearch
+// // to ignore any fields that are not present.
+//
+// final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
+// final FieldSortBuilder stringSort = SortBuilders.fieldSort(stringFieldName)
+// .order(order).ignoreUnmapped(true);
+// srb.addSort(stringSort);
+//
+// logger.debug(" Sort: {} order by {}", stringFieldName, order.toString());
+//
+// final String longFieldName = LONG_PREFIX + sp.getPropertyName();
+// final FieldSortBuilder longSort = SortBuilders.fieldSort(longFieldName)
+// .order(order).ignoreUnmapped(true);
+// srb.addSort(longSort);
+// logger.debug(" Sort: {} order by {}", longFieldName, order.toString());
+//
+//
+// final String doubleFieldName = DOUBLE_PREFIX + sp.getPropertyName();
+// final FieldSortBuilder doubleSort = SortBuilders.fieldSort(doubleFieldName)
+// .order(order).ignoreUnmapped(true);
+// srb.addSort(doubleSort);
+// logger.debug(" Sort: {} order by {}", doubleFieldName, order.toString());
+//
+//
+// final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
+// final FieldSortBuilder booleanSort = SortBuilders.fieldSort(booleanFieldName)
+// .order(order).ignoreUnmapped(true);
+// srb.addSort(booleanSort);
+// logger.debug(" Sort: {} order by {}", booleanFieldName, order.toString());
}
return srb;
}
@@ -168,7 +165,7 @@ public class SearchRequestBuilderStrategy {
//make sure we have entity in the context
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
- boolQueryBuilder.must( QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ) );
+ boolQueryBuilder.must( QueryBuilders.termQuery( IndexingUtils.EDGE_SEARCH_FIELDNAME, context ) );
boolQueryBuilder.must( queryBuilder );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
index 3efad3b..c60de42 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
@@ -64,7 +64,7 @@ public class LegacyIndexIdentifier implements FailureMonitorImpl.IndexIdentifier
*/
private String getIndexBase() {
StringBuilder sb = new StringBuilder();
- sb.append(config.getIndexPrefix()).append(IndexingUtils.SEPARATOR);
+ sb.append(config.getIndexPrefix()).append(IndexingUtils.FIELD_SEPERATOR );
IndexingUtils.idString(sb, applicationScope.getApplication());
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
index 587eab3..7b13b66 100644
--- a/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
+++ b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
@@ -3,57 +3,90 @@
"properties": {
"entityId": {
"type": "string",
- "index": "not_analyzed"
+ "index": "not_analyzed",
+ "doc_values": true
},
"entityVersion": {
"type": "string",
- "index": "not_analyzed"
+ "index": "not_analyzed",
+ "doc_values": true
},
"entityType": {
"type": "string",
- "index": "not_analyzed"
+ "index": "not_analyzed",
+ "doc_values": true
},
"applicationId": {
"type": "string",
- "index": "not_analyzed"
+ "index": "not_analyzed",
+ "doc_values": true
},
"edgeNodeId": {
"type": "string",
- "index": "not_analyzed"
+ "index": "not_analyzed",
+ "doc_values": true
},
"edgeName": {
"type": "string",
- "index": "not_analyzed"
+ "index": "not_analyzed",
+ "doc_values": true
},
"edgeType": {
"type": "string",
- "index": "not_analyzed"
+ "index": "not_analyzed",
+ "doc_values": true
+ },
+ "edgeTimestamp": {
+ "type": "long",
+ "doc_values": true
},
- "edgeTimestamp": {"type": "long"},
"edgeSearch": {
"type": "string",
- "index": "not_analyzed"
+ "index": "not_analyzed",
+ "doc_values": true
},
"fields": {
"type": "nested",
"properties": {
"name": {
"type": "string",
- "index": "not_analyzed"
+ "index": "not_analyzed",
+ "doc_values": true
+ },
+ "boolean": {
+ "type": "boolean",
+ "doc_values": true
+ },
+ "int": {
+ "type": "integer",
+ "doc_values": true
+ },
+ "long": {
+ "type": "long",
+ "doc_values": true
+ },
+ "float": {
+ "type": "float",
+ "doc_values": true
+ },
+ "double": {
+ "type": "double",
+ "doc_values": true
+ },
+ "location": {
+ "type": "geo_point",
+ "lat_lon": true,
+ "geohash": true,
+ "doc_values": true
},
- "boolean": {"type": "boolean"},
- "short": {"type": "short"},
- "int": {"type": "integer"},
- "long": {"type": "long"},
- "float": {"type": "float"},
- "double": {"type": "double"},
"string": {
"type": "string",
"fields": {
"exact": {
"type": "string",
"index": "not_analyzed"
- }
+ },
+ "doc_values": true
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 0bdf1ca..7aa6d4f 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -133,7 +133,7 @@ public class EntityIndexTest extends BaseIT {
EntityIndexBatch batch = entityIndex.createBatch();
Entity entity = new Entity( entityType );
EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
- entity.setField( new UUIDField( IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID() ) );
+ entity.setField( new UUIDField( IndexingUtils.ENTITY_ID_FIELDNAME, UUID.randomUUID() ) );
entity.setField( new StringField( "testfield", "test" ) );
batch.index( indexEdge, entity );
batch.execute().get();
@@ -327,7 +327,7 @@ public class EntityIndexTest extends BaseIT {
Entity entity = new Entity( entityType );
entity = EntityIndexMapUtils.fromMap( entity, item );
EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
- entity.setField( new UUIDField( IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID() ) );
+ entity.setField( new UUIDField( IndexingUtils.ENTITY_ID_FIELDNAME, UUID.randomUUID() ) );
batch.index( indexEdge, entity );
batch.execute().get();
@@ -365,7 +365,7 @@ public class EntityIndexTest extends BaseIT {
Entity entity = EntityIndexMapUtils.fromMap( entityMap );
EntityUtils.setId( entity, new SimpleId( "fastcar" ) );
EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
- entity.setField( new UUIDField( IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID() ) );
+ entity.setField( new UUIDField( IndexingUtils.ENTITY_ID_FIELDNAME, UUID.randomUUID() ) );
entityIndex.createBatch().index( searchEdge, entity ).execute().get();
ei.refresh();
@@ -653,7 +653,7 @@ public class EntityIndexTest extends BaseIT {
EntityUtils.setId( user, userId );
EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
- user.setField( new UUIDField( IndexingUtils.ENTITYID_ID_FIELDNAME, UUIDGenerator.newTimeUUID() ) );
+ user.setField( new UUIDField( IndexingUtils.ENTITY_ID_FIELDNAME, UUIDGenerator.newTimeUUID() ) );
entities.add( userId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1e3ea744/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverterTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverterTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverterTest.java
new file mode 100644
index 0000000..c198020
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverterTest.java
@@ -0,0 +1,351 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.IndexEdge;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.field.ArrayField;
+import org.apache.usergrid.persistence.model.field.BooleanField;
+import org.apache.usergrid.persistence.model.field.DoubleField;
+import org.apache.usergrid.persistence.model.field.Field;
+import org.apache.usergrid.persistence.model.field.FloatField;
+import org.apache.usergrid.persistence.model.field.IntegerField;
+import org.apache.usergrid.persistence.model.field.LocationField;
+import org.apache.usergrid.persistence.model.field.LongField;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.field.value.Location;
+import org.apache.usergrid.persistence.model.util.EntityUtils;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import rx.functions.Action2;
+
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
+import static org.junit.Assert.assertEquals;
+
+
+/**
+ * Tests our entity conversion
+ */
+public class EntityToMapConverterTest {
+
+
+ @Test
+ public void testBaseFields() {
+ Entity entity = new Entity( "test" );
+
+
+ final UUID version = UUIDGenerator.newTimeUUID();
+
+ EntityUtils.setVersion( entity, version );
+
+ final ApplicationScope scope = new ApplicationScopeImpl( createId( "application" ) );
+
+ final IndexEdge indexEdge =
+ new IndexEdgeImpl( createId( "source" ), "testEdgeType", SearchEdge.NodeType.SOURCE, 1000 );
+
+
+ final Map<String, Object> entityMap = EntityToMapConverter.convert( scope, indexEdge, entity );
+
+
+ //verify our root fields
+
+
+ final String applicationId = entityMap.get( IndexingUtils.APPLICATION_ID_FIELDNAME ).toString();
+
+ assertEquals( IndexingUtils.idString( scope.getApplication() ), applicationId );
+
+ final String entityIdString = entityMap.get( IndexingUtils.ENTITY_ID_FIELDNAME ).toString();
+
+ assertEquals( IndexingUtils.idString( entity.getId() ), entityIdString );
+
+
+ final String versionString = entityMap.get( IndexingUtils.ENTITY_VERSION_FIELDNAME ).toString();
+
+ assertEquals( versionString, version.toString() );
+
+ final String entityTypeString = entityMap.get( IndexingUtils.ENTITY_TYPE_FIELDNAME ).toString();
+
+ assertEquals( IndexingUtils.getType( scope, entity.getId() ), entityTypeString );
+
+ final String nodeIdString = entityMap.get( IndexingUtils.EDGE_NODE_ID_FIELDNAME ).toString();
+
+ assertEquals( IndexingUtils.idString( indexEdge.getNodeId() ), nodeIdString );
+
+ final String edgeName = entityMap.get( IndexingUtils.EDGE_NAME_FIELDNAME ).toString();
+
+ assertEquals( indexEdge.getEdgeName(), edgeName );
+
+ final String nodeType = entityMap.get( IndexingUtils.EDGE_NODE_TYPE_FIELDNAME ).toString();
+
+
+ assertEquals( indexEdge.getNodeType().toString(), nodeType );
+
+ final long edgeTimestamp = ( long ) entityMap.get( IndexingUtils.EDGE_TIMESTAMP_FIELDNAME );
+
+ assertEquals( indexEdge.getTimestamp(), edgeTimestamp );
+
+ final String edgeSearch = entityMap.get( IndexingUtils.EDGE_SEARCH_FIELDNAME ).toString();
+
+ assertEquals( IndexingUtils.createContextName( scope, indexEdge ), edgeSearch );
+
+
+ final List<EntityField> fields = ( List<EntityField> ) entityMap.get( IndexingUtils.ENTITY_FIELDS );
+
+ assertEquals( 0, fields.size() );
+ }
+
+
+ @Test
+ public void testStringField() {
+ testSingleField( new StringField( "name", "value" ),
+ ( field, entityField ) -> assertEquals( field.getValue(),
+ entityField.get( IndexingUtils.FIELD_STRING ) ) );
+ }
+
+
+ @Test
+ public void testBooleanField() {
+ testSingleField( new BooleanField( "name", true ),
+ ( field, entityField ) -> assertEquals( field.getValue(),
+ entityField.get( IndexingUtils.FIELD_BOOLEAN ) ) );
+ }
+
+
+ @Test
+ public void testIntegerField() {
+ testSingleField( new IntegerField( "name", 100 ),
+ ( field, entityField ) -> assertEquals( field.getValue(),
+ entityField.get( IndexingUtils.FIELD_INT ) ) );
+ }
+
+
+ @Test
+ public void testLongField() {
+ testSingleField( new LongField( "name", 100l ),
+ ( field, entityField ) -> assertEquals( field.getValue(),
+ entityField.get( IndexingUtils.FIELD_LONG ) ) );
+ }
+
+
+ @Test
+ public void testFloadField() {
+ testSingleField( new FloatField( "name", 1.10f ),
+ ( field, entityField ) -> assertEquals( field.getValue(),
+ entityField.get( IndexingUtils.FIELD_FLOAT ) ) );
+ }
+
+
+ @Test
+ public void testDoubleField() {
+ testSingleField( new DoubleField( "name", 2.20d ),
+ ( field, entityField ) -> assertEquals( field.getValue(),
+ entityField.get( IndexingUtils.FIELD_DOUBLE ) ) );
+ }
+
+
+ @Test
+ public void testLocationField() {
+ testSingleField( new LocationField( "name", new Location( 10, 20 ) ), (field, entityField) -> {
+ final Map<String, Double> latLong = ( Map<String, Double> ) entityField.get( IndexingUtils.FIELD_LOCATION );
+
+ assertEquals(Double.valueOf( 10 ), latLong.get( org.apache.usergrid.persistence.model.entity.EntityToMapConverter.LAT ));
+ assertEquals(Double.valueOf( 20 ), latLong.get( org.apache.usergrid.persistence.model.entity.EntityToMapConverter.LON ));
+ } );
+ }
+
+
+ private static void assertField() {
+
+ }
+
+
+ /**
+ * Test the single field in our root level
+ */
+ public <T> void testSingleField( final Field<T> field,
+ Action2<Field, EntityField> assertFunction ) {
+ Entity entity = new Entity( "test" );
+ entity.setField( field );
+
+
+ final UUID version = UUIDGenerator.newTimeUUID();
+
+ EntityUtils.setVersion( entity, version );
+
+ final ApplicationScope scope = new ApplicationScopeImpl( createId( "application" ) );
+
+ final IndexEdge indexEdge =
+ new IndexEdgeImpl( createId( "source" ), "testEdgeType", SearchEdge.NodeType.SOURCE, 1000 );
+
+
+ final Map<String, Object> entityMap = EntityToMapConverter.convert( scope, indexEdge, entity );
+
+
+ final List<EntityField> fields = ( List<EntityField> ) entityMap.get( IndexingUtils.ENTITY_FIELDS );
+
+ assertEquals( 1, fields.size() );
+
+ final EntityField esField = fields.get( 0 );
+
+ assertEquals( field.getName(), esField.get( IndexingUtils.FIELD_NAME ) );
+
+ assertFunction.call( field, esField );
+ }
+
+
+ @Test
+ public void testStringArray() {
+
+
+ final ArrayField<String> array = new ArrayField<>( "strings" );
+
+ array.add( "one" );
+ array.add( "two" );
+ array.add( "three" );
+
+ testPrimitiveArray( array, IndexingUtils.FIELD_STRING );
+ }
+
+
+ @Test
+ public void testBooleanArray() {
+
+
+ final ArrayField<Boolean> array = new ArrayField<>( "bools" );
+
+ array.add( true );
+ array.add( true );
+ array.add( false );
+ array.add( true );
+
+ testPrimitiveArray( array, IndexingUtils.FIELD_BOOLEAN );
+ }
+
+
+ @Test
+ public void testIntArray() {
+
+
+ final ArrayField<Integer> array = new ArrayField<>( "ints" );
+
+ array.add( 1 );
+ array.add( 2 );
+ array.add( 3 );
+
+ testPrimitiveArray( array, IndexingUtils.FIELD_INT );
+ }
+
+
+ @Test
+ public void testLongArray() {
+
+
+ final ArrayField<Long> array = new ArrayField<>( "longs" );
+
+ array.add( 1l );
+ array.add( 2l );
+ array.add( 3l );
+
+ testPrimitiveArray( array, IndexingUtils.FIELD_LONG );
+ }
+
+
+ @Test
+ public void testFloatArray() {
+
+
+ final ArrayField<Float> array = new ArrayField<>( "floats" );
+
+ array.add( 1.0f );
+ array.add( 2.0f );
+ array.add( 3.0f );
+
+ testPrimitiveArray( array, IndexingUtils.FIELD_FLOAT );
+ }
+
+
+ @Test
+ public void testDoubleArray() {
+
+
+ final ArrayField<Double> array = new ArrayField<>( "doubles" );
+
+ array.add( 1.0d );
+ array.add( 2.0d );
+ array.add( 3.0d );
+
+ testPrimitiveArray( array, IndexingUtils.FIELD_DOUBLE );
+ }
+
+
+ /**
+ * Test primitive arrays in the root of an object
+ *
+ * @param indexType the field name for the expected type in ES
+ */
+ private <T> void testPrimitiveArray( final ArrayField<T> array, final String indexType ) {
+
+ Entity entity = new Entity( "test" );
+
+
+ entity.setField( array );
+
+
+ final UUID version = UUIDGenerator.newTimeUUID();
+
+ EntityUtils.setVersion( entity, version );
+
+ final ApplicationScope scope = new ApplicationScopeImpl( createId( "application" ) );
+
+ final IndexEdge indexEdge =
+ new IndexEdgeImpl( createId( "source" ), "testEdgeType", SearchEdge.NodeType.SOURCE, 1000 );
+
+
+ final Map<String, Object> entityMap = EntityToMapConverter.convert( scope, indexEdge, entity );
+
+
+ final List<EntityField> fields = ( List<EntityField> ) entityMap.get( IndexingUtils.ENTITY_FIELDS );
+
+ assertEquals( array.size(), fields.size() );
+
+
+ for ( int i = 0; i < array.size(); i++ ) {
+ final EntityField field = fields.get( i );
+
+ assertEquals( array.getName(), field.get( IndexingUtils.FIELD_NAME ) );
+
+ assertEquals( array.getValue().get( i ), field.get( indexType ) );
+ }
+ }
+}