You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/09/25 19:49:53 UTC
[1/2] git commit: Name ES index fields with prefixes to indicate
types of String (su), String Analyzed (sa), Geo (go),
Number (nu) and Boolean (bu),
so that we are able to support multi-typed fields required by Usergrid.
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o 4c9b6952d -> b9febe910
Name ES index fields with prefixes to indicate types of String (su), String Analyzed (sa), Geo (go), Number (nu) and Boolean (bu), so that we are able to support multi-typed fields required by Usergrid.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b8facb6d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b8facb6d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b8facb6d
Branch: refs/heads/two-dot-o
Commit: b8facb6d1dadfc8e4f73a6edf9819c6c1fa1c508
Parents: 4c9b695
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Sep 25 13:47:55 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Sep 25 13:47:55 2014 -0400
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 79 ++++++++----
.../persistence/index/impl/EsQueryVistor.java | 128 +++++++++++++++----
2 files changed, 156 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b8facb6d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 58c3726..5fb787c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -18,7 +18,6 @@
package org.apache.usergrid.persistence.index.impl;
-import com.google.common.base.Joiner;
import com.google.common.util.concurrent.AtomicDouble;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
@@ -45,12 +44,18 @@ import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.field.ArrayField;
+import org.apache.usergrid.persistence.model.field.BooleanField;
+import org.apache.usergrid.persistence.model.field.DoubleField;
import org.apache.usergrid.persistence.model.field.EntityObjectField;
import org.apache.usergrid.persistence.model.field.Field;
+import org.apache.usergrid.persistence.model.field.FloatField;
+import org.apache.usergrid.persistence.model.field.IntegerField;
import org.apache.usergrid.persistence.model.field.ListField;
import org.apache.usergrid.persistence.model.field.LocationField;
+import org.apache.usergrid.persistence.model.field.LongField;
import org.apache.usergrid.persistence.model.field.SetField;
import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.field.UUIDField;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -98,8 +103,11 @@ public class EsEntityIndexImpl implements EntityIndex {
private final AtomicLong indexedCount = new AtomicLong(0L);
private final AtomicDouble averageIndexTime = new AtomicDouble(0);
- public static final String ANALYZED_SUFFIX = "_ug_analyzed";
- public static final String GEO_SUFFIX = "_ug_geo";
+ public static final String STRING_PREFIX = "su_";
+ public static final String ANALYZED_STRING_PREFIX = "sa_";
+ public static final String GEO_PREFIX = "go_";
+ public static final String NUMBER_PREFIX = "nu_";
+ public static final String BOOLEAN_PREFIX = "bu_";
public static final String ENTITYID_FIELDNAME = "zzz_entityid_zzz";
@@ -281,7 +289,10 @@ public class EsEntityIndexImpl implements EntityIndex {
}
Map<String, Object> entityAsMap = EsEntityIndexImpl.entityToMap(entity);
- entityAsMap.put(ENTITYID_FIELDNAME,entity.getId().getUuid().toString());
+
+ // need prefix here becuase we index UUIDs as strings
+ entityAsMap.put( STRING_PREFIX + ENTITYID_FIELDNAME,
+ entity.getId().getUuid().toString().toLowerCase());
// let caller add these fields if needed
// entityAsMap.put("created", entity.getId().getUuid().timestamp();
@@ -454,8 +465,13 @@ public class EsEntityIndexImpl implements EntityIndex {
/**
- * Convert Entity to Map, adding version_ug_field and a {name}_ug_analyzed field for each
- * StringField.
+ * Convert Entity to Map. Adding prefixes for types:
+ *
+ * su_ - String unanalyzed field
+ * sa_ - String analyzed field
+ * go_ - Location field
+ * nu_ - Number field
+ * bu_ - Boolean field
*/
private static Map entityToMap(EntityObject entity) {
@@ -466,18 +482,8 @@ public class EsEntityIndexImpl implements EntityIndex {
if (f instanceof ListField) {
List list = (List) field.getValue();
- entityMap.put(field.getName().toLowerCase(),
- new ArrayList(processCollectionForMap(list)));
-
- if ( !list.isEmpty() ) {
- if ( list.get(0) instanceof String ) {
- Joiner joiner = Joiner.on(" ").skipNulls();
- String joined = joiner.join(list);
- entityMap.put(field.getName().toLowerCase() + ANALYZED_SUFFIX,
- new ArrayList(processCollectionForMap(list)));
-
- }
- }
+ entityMap.put(field.getName().toLowerCase(),
+ new ArrayList(processCollectionForMap(list)));
} else if (f instanceof ArrayField) {
List list = (List) field.getValue();
@@ -496,9 +502,10 @@ public class EsEntityIndexImpl implements EntityIndex {
} else if (f instanceof StringField) {
// index in lower case because Usergrid queries are case insensitive
- entityMap.put(field.getName().toLowerCase(), ((String) field.getValue()).toLowerCase());
- entityMap.put(field.getName().toLowerCase()
- + ANALYZED_SUFFIX, ((String) field.getValue()).toLowerCase());
+ entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
+ ((String) field.getValue()).toLowerCase());
+ entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
+ ((String) field.getValue()).toLowerCase());
} else if (f instanceof LocationField) {
LocationField locField = (LocationField) f;
@@ -507,7 +514,27 @@ public class EsEntityIndexImpl implements EntityIndex {
// field names lat and lon trigger ElasticSearch geo location
locMap.put("lat", locField.getValue().getLatitude());
locMap.put("lon", locField.getValue().getLongtitude());
- entityMap.put(field.getName().toLowerCase() + GEO_SUFFIX, locMap);
+ entityMap.put( GEO_PREFIX + field.getName().toLowerCase(), locMap);
+
+ } else if ( f instanceof BooleanField ) {
+
+ entityMap.put( NUMBER_PREFIX + field.getName().toLowerCase(), field.getValue());
+
+ } else if ( f instanceof DoubleField
+ || f instanceof FloatField
+ || f instanceof IntegerField
+ || f instanceof LongField ) {
+
+ entityMap.put( NUMBER_PREFIX + field.getName().toLowerCase(), field.getValue());
+
+ } else if ( f instanceof BooleanField ) {
+
+ entityMap.put( BOOLEAN_PREFIX + field.getName().toLowerCase(), field.getValue());
+
+ } else if ( f instanceof UUIDField ) {
+
+ entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
+ field.getValue().toString() );
} else {
entityMap.put(field.getName().toLowerCase(), field.getValue());
@@ -570,10 +597,10 @@ public class EsEntityIndexImpl implements EntityIndex {
.startObject(type)
.startArray("dynamic_templates")
- // any string with field name that ends with _ug_analyzed gets analyzed
+ // any string with field name that starts with sa_ gets analyzed
.startObject()
.startObject("template_1")
- .field("match", "*" + ANALYZED_SUFFIX)
+ .field("match", ANALYZED_STRING_PREFIX + "*")
.field("match_mapping_type", "string")
.startObject("mapping")
.field("type", "string")
@@ -594,10 +621,10 @@ public class EsEntityIndexImpl implements EntityIndex {
.endObject()
.endObject()
- // fields location_ug_geo get geo-indexed
+ // fields names starting with go_ get geo-indexed
.startObject()
.startObject("template_3")
- .field("match", "location" + GEO_SUFFIX)
+ .field("match", GEO_PREFIX + "location")
.startObject("mapping")
.field("type", "geo_point")
.endObject()
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b8facb6d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
index 21e7b1a..07f98a2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
@@ -18,12 +18,19 @@
package org.apache.usergrid.persistence.index.impl;
+import com.google.common.base.Joiner;
import java.util.ArrayList;
import java.util.List;
import java.util.Stack;
+import java.util.UUID;
import org.apache.usergrid.persistence.index.exceptions.NoFullTextIndexException;
import org.apache.usergrid.persistence.index.exceptions.NoIndexException;
import org.apache.usergrid.persistence.index.exceptions.IndexException;
+import static org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.ANALYZED_STRING_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.BOOLEAN_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.GEO_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.NUMBER_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.STRING_PREFIX;
import org.apache.usergrid.persistence.index.query.tree.AndOperand;
import org.apache.usergrid.persistence.index.query.tree.ContainsOperand;
import org.apache.usergrid.persistence.index.query.tree.Equal;
@@ -55,6 +62,7 @@ public class EsQueryVistor implements QueryVisitor {
Stack<QueryBuilder> stack = new Stack<QueryBuilder>();
List<FilterBuilder> filterBuilders = new ArrayList<FilterBuilder>();
+
@Override
public void visit( AndOperand op ) throws IndexException {
@@ -99,6 +107,7 @@ public class EsQueryVistor implements QueryVisitor {
stack.push( qb );
}
+
@Override
public void visit( OrOperand op ) throws IndexException {
@@ -125,6 +134,7 @@ public class EsQueryVistor implements QueryVisitor {
stack.push( qb );
}
+
@Override
public void visit( NotOperand op ) throws IndexException {
op.getOperation().visit( this );
@@ -134,17 +144,17 @@ public class EsQueryVistor implements QueryVisitor {
}
}
+
@Override
public void visit( ContainsOperand op ) throws NoFullTextIndexException {
String name = op.getProperty().getValue();
name = name.toLowerCase();
Object value = op.getLiteral().getValue();
- if ( value instanceof String ) {
- name = addAnayzedSuffix( name );
- }
+ name = addPrefix( value, name, true );
stack.push( QueryBuilders.matchQuery( name, value ));
}
+
@Override
public void visit( WithinOperand op ) {
@@ -155,8 +165,8 @@ public class EsQueryVistor implements QueryVisitor {
float lon = op.getLongitude().getFloatValue();
float distance = op.getDistance().getFloatValue();
- if ( !name.endsWith( EsEntityIndexImpl.GEO_SUFFIX )) {
- name += EsEntityIndexImpl.GEO_SUFFIX;
+ if ( !name.startsWith( GEO_PREFIX )) {
+ name = GEO_PREFIX + name;
}
FilterBuilder fb = FilterBuilders.geoDistanceFilter( name )
@@ -164,28 +174,27 @@ public class EsQueryVistor implements QueryVisitor {
filterBuilders.add( fb );
}
+
@Override
public void visit( LessThan op ) throws NoIndexException {
String name = op.getProperty().getValue();
name = name.toLowerCase();
Object value = op.getLiteral().getValue();
- if ( value instanceof String ) {
- name = addAnayzedSuffix( name );
- }
+ name = addPrefix( value, name );
stack.push( QueryBuilders.rangeQuery( name ).lt( value ));
}
+
@Override
public void visit( LessThanEqual op ) throws NoIndexException {
String name = op.getProperty().getValue();
name = name.toLowerCase();
Object value = op.getLiteral().getValue();
- if ( value instanceof String ) {
- name = addAnayzedSuffix( name );
- }
+ name = addPrefix( value, name );
stack.push( QueryBuilders.rangeQuery( name ).lte( value ));
}
+
@Override
public void visit( Equal op ) throws NoIndexException {
String name = op.getProperty().getValue();
@@ -194,48 +203,116 @@ public class EsQueryVistor implements QueryVisitor {
if ( value instanceof String ) {
String svalue = (String)value;
+
+ // use normal prefix, we need unanalyzed field for equals op
+ name = addPrefix( value, name );
if ( svalue.indexOf("*") != -1 ) {
- // for wildcard expression we need analuzed field, add suffix
- //name = addAnayzedSuffix( name );
stack.push( QueryBuilders.wildcardQuery(name, svalue) );
return;
}
- // for equal operation on string, need to use unanalyzed field, leave off the suffix
- value = svalue;
- }
+ } else {
+ name = addPrefix( value, name );
+ }
stack.push( QueryBuilders.termQuery( name, value ));
}
+
@Override
public void visit( GreaterThan op ) throws NoIndexException {
String name = op.getProperty().getValue();
name = name.toLowerCase();
Object value = op.getLiteral().getValue();
- if ( value instanceof String ) {
- name = addAnayzedSuffix( name );
- }
+ name = addPrefix( value, name );
stack.push( QueryBuilders.rangeQuery( name ).gt( value ) );
}
+
@Override
public void visit( GreaterThanEqual op ) throws NoIndexException {
String name = op.getProperty().getValue();
name = name.toLowerCase();
Object value = op.getLiteral().getValue();
- if ( value instanceof String ) {
- name = addAnayzedSuffix( name );
- }
+ name = addPrefix( value, name );
stack.push( QueryBuilders.rangeQuery( name ).gte( value ) );
}
- private String addAnayzedSuffix( String name ) {
- if ( name.endsWith(EsEntityIndexImpl.ANALYZED_SUFFIX) ) {
+
+ private String addPrefix( Object value, String origname ) {
+ return addPrefix(value, origname, false);
+ }
+
+
+ private String addPrefix( Object value, String origname, boolean analyzed ) {
+
+ String name = origname;
+
+ // logic to deal with nested property names
+ // only add prefix to last name in property
+
+ String[] parts = origname.split("\\.");
+ if ( parts.length > 1 ) {
+ name = parts[ parts.length - 1 ];
+ }
+
+ if ( value instanceof String && analyzed ) {
+ name = addAnalyzedStringPrefix( name );
+
+ } else if ( value instanceof String ) {
+ name = addStringPrefix( name );
+
+ } else if ( value instanceof Number ) {
+ name = addNumberPrefix(name);
+
+ } else if ( value instanceof Boolean ) {
+ name = addBooleanPrefix(name);
+
+ } else if ( value instanceof UUID ) {
+ name = addStringPrefix(name);
+ }
+
+ if ( parts.length > 1 ) {
+ parts[parts.length - 1] = name;
+ Joiner joiner = Joiner.on(".").skipNulls();
+ return joiner.join(parts);
+ }
+
+ return name;
+ }
+
+
+ private String addAnalyzedStringPrefix( String name ) {
+ if ( name.startsWith( ANALYZED_STRING_PREFIX ) ) {
+ return name;
+ }
+ return ANALYZED_STRING_PREFIX + name;
+ }
+
+
+ private String addStringPrefix( String name ) {
+ if ( name.startsWith( STRING_PREFIX ) ) {
+ return name;
+ }
+ return STRING_PREFIX + name;
+ }
+
+
+ private String addNumberPrefix( String name ) {
+ if ( name.startsWith( NUMBER_PREFIX ) ) {
+ return name;
+ }
+ return NUMBER_PREFIX + name;
+ }
+
+
+ private String addBooleanPrefix( String name ) {
+ if ( name.startsWith( BOOLEAN_PREFIX ) ) {
return name;
}
- return name + EsEntityIndexImpl.ANALYZED_SUFFIX;
+ return BOOLEAN_PREFIX + name;
}
+
@Override
public QueryBuilder getQueryBuilder() {
@@ -245,6 +322,7 @@ public class EsQueryVistor implements QueryVisitor {
return stack.pop();
}
+
@Override
public FilterBuilder getFilterBuilder() {
[2/2] git commit: Added couple of interesting paths, commented out.
Posted by sn...@apache.org.
Added couple of interesting paths, commented out.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b9febe91
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b9febe91
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b9febe91
Branch: refs/heads/two-dot-o
Commit: b9febe910bc02465a41802029dbbd55cc2b2170a
Parents: b8facb6
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Sep 25 13:49:40 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Sep 25 13:49:40 2014 -0400
----------------------------------------------------------------------
stack/core/src/test/resources/log4j.properties | 1 +
stack/rest/src/test/resources/log4j.properties | 7 +++++--
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b9febe91/stack/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/log4j.properties b/stack/core/src/test/resources/log4j.properties
index 0d98883..51f6112 100644
--- a/stack/core/src/test/resources/log4j.properties
+++ b/stack/core/src/test/resources/log4j.properties
@@ -39,6 +39,7 @@ log4j.logger.org.apache.usergrid.rest.security.AllowAjaxFilter=WARN, stdout
log4j.logger.me.prettyprint.hector.api.beans.AbstractComposite=ERROR, stdout
log4j.logger.org.apache.usergrid.locking.singlenode.SingleNodeLockManagerImpl=DEBUG, stdout
+#log4j.logger.org.apache.usergrid.persistence.PerformanceEntityReadTest=DEBUG
#log4j.logger.org.apache.usergrid.persistence=INFO
#log4j.logger.org.apache.usergrid.corepersistence=DEBUG
#log4j.logger.com.netflix.hystrix=DEBUG
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b9febe91/stack/rest/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/log4j.properties b/stack/rest/src/test/resources/log4j.properties
index dd3c9fb..8b1eac2 100644
--- a/stack/rest/src/test/resources/log4j.properties
+++ b/stack/rest/src/test/resources/log4j.properties
@@ -18,7 +18,7 @@
# and the pattern to %c instead of %l. (%l is slower.)
# output messages into a rolling log file as well as stdout
-log4j.rootLogger=ERROR,stdout
+log4j.rootLogger=WARN,stdout
# stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
@@ -44,6 +44,8 @@ log4j.logger.org.apache.usergrid.locking.singlenode.SingleNodeLockManagerImpl=DE
log4j.logger.org.apache.usergrid.rest.filters.MeteringFilter=ERROR
#log4j.logger.org.apache.usergrid.corepersistence=DEBUG
+#log4j.logger.org.apache.usergrid.corepersistence.CpRelationManager=DEBUG
+#log4j.logger.org.apache.usergrid.corepersistence.CpEntityManager=DEBUG
#log4j.logger.org.apache.usergrid.corepersistence.CpEntityManagerFactory=DEBUG
#log4j.logger.org.apache.usergrid.corepersistence.CpSetup=INFO
@@ -54,7 +56,8 @@ log4j.logger.org.apache.usergrid.rest.filters.MeteringFilter=ERROR
#log4j.logger.org.apache.usergrid.services=DEBUG
#log4j.logger.org.apache.cassandra.service.StorageProxy=DEBUG, stdout
-#log4j.logger.org.apache.usergrid.rest=DEBUG
+#log4j.logger.org.apache.usergrid.rest=INFO
+#log4j.logger.org.apache.usergrid.rest.NotificationsIT=DEBUG
#log4j.logger.org.apache.usergrid.services.notifications.ApplicationQueueManager=DEBUG
#log4j.logger.org.apache.usergrid.services.notifications.SingleQueueTaskManager=DEBUG