You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/09/24 21:21:43 UTC
[1/2] git commit: Work in progress. Mapping utils need changed.
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o-candidate-serialization 101ac4fd3 -> 363375d9c
Work in progress. Mapping utils need changed.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2c66c864
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2c66c864
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2c66c864
Branch: refs/heads/two-dot-o-candidate-serialization
Commit: 2c66c864596982b32c4088c413cfaee10f741311
Parents: 2e4fc15
Author: Todd Nine <to...@apache.org>
Authored: Wed Sep 24 13:21:26 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Wed Sep 24 13:21:26 2014 -0600
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 15 ++--
.../persistence/index/impl/EsQueryVistor.java | 90 ++++++++++++--------
.../index/impl/EntityIndexMapUtils.java | 1 -
3 files changed, 59 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c66c864/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index d309d60..87fdc11 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -91,7 +91,6 @@ public class EsEntityIndexImpl implements EntityIndex {
private final IndexScope indexScope;
private final Client client;
- private final SerializationFig serializationFig;
protected EntityCollectionManagerFactory ecmFactory;
@@ -101,7 +100,6 @@ public class EsEntityIndexImpl implements EntityIndex {
private final AtomicLong indexedCount = new AtomicLong(0L);
private final AtomicDouble averageIndexTime = new AtomicDouble(0);
- public static final String ANALYZED_PREFIX = "ug_analyzed_s";
public static final String GEO_SUFFIX = "location";
// public static final String COLLECTION_SCOPE_FIELDNAME = "zzz__collectionscope__zzz";
@@ -122,8 +120,7 @@ public class EsEntityIndexImpl implements EntityIndex {
@Assisted final IndexScope indexScope,
IndexFig config,
EsProvider provider,
- EntityCollectionManagerFactory factory,
- SerializationFig serializationFig
+ EntityCollectionManagerFactory factory
) {
IndexValidationUtils.validateIndexScope( indexScope );
@@ -137,7 +134,6 @@ public class EsEntityIndexImpl implements EntityIndex {
this.indexName = createIndexName( config.getIndexPrefix(), indexScope);
this.indexType = createCollectionScopeTypeName( indexScope );
- this.serializationFig = serializationFig;
this.refresh = config.isForcedRefresh();
this.cursorTimeout = config.getQueryCursorTimeout();
@@ -490,7 +486,7 @@ public class EsEntityIndexImpl implements EntityIndex {
if ( list.get(0) instanceof String ) {
Joiner joiner = Joiner.on(" ").skipNulls();
String joined = joiner.join(list);
- entityMap.put(Types.STRING.PREFIX+ ANALYZED_PREFIX +field.getName().toLowerCase(),
+ entityMap.put(Types.STRING_ANALYZED.PREFIX +field.getName().toLowerCase(),
new ArrayList(processCollectionForMap(list)));
}
@@ -514,7 +510,7 @@ public class EsEntityIndexImpl implements EntityIndex {
// index in lower case because Usergrid queries are case insensitive
entityMap.put(Types.STRING.PREFIX+field.getName().toLowerCase(), ((String) field.getValue()).toLowerCase());
- entityMap.put(Types.STRING.PREFIX+ ANALYZED_PREFIX +field.getName().toLowerCase(), ((String) field.getValue()).toLowerCase());
+ entityMap.put(Types.STRING.STRING_ANALYZED.PREFIX +field.getName().toLowerCase(), ((String) field.getValue()).toLowerCase());
} else if (f instanceof LocationField) {
LocationField locField = (LocationField) f;
@@ -589,7 +585,7 @@ public class EsEntityIndexImpl implements EntityIndex {
// any string with field name that ends with _ug_analyzed gets analyzed
.startObject()
.startObject("template_1")
- .field("match", Types.STRING.PREFIX + ANALYZED_PREFIX +"*")
+ .field("match", Types.STRING_ANALYZED.PREFIX + "*")
.field("match_mapping_type", "string")
.startObject("mapping")
.field("type", "string")
@@ -613,7 +609,7 @@ public class EsEntityIndexImpl implements EntityIndex {
// fields location_ug_geo get geo-indexed
.startObject()
.startObject("template_3")
- .field("match", Types.LOCATION + "*")
+ .field("match", Types.LOCATION.PREFIX + "*")
.startObject("mapping")
.field("type", "geo_point")
.endObject()
@@ -642,6 +638,7 @@ public class EsEntityIndexImpl implements EntityIndex {
}
public static enum Types{
STRING("s"),
+ STRING_ANALYZED("sa"),
LIST("l"),
LOCATION("loc"),
ARRAY("a"),
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c66c864/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
index 84b8eeb..d7f94cc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
@@ -136,12 +136,10 @@ public class EsQueryVistor implements QueryVisitor {
@Override
public void visit( ContainsOperand op ) throws NoFullTextIndexException {
- String name = op.getProperty().getValue();
- name = name.toLowerCase();
+ String name = EsEntityIndexImpl.Types.STRING_ANALYZED + op.getProperty().getValue().toLowerCase();
Object value = op.getLiteral().getValue();
- if ( value instanceof String ) {
- name = addAnayzedSuffix( name );
- }
+
+
stack.push( QueryBuilders.matchQuery( name, value ));
}
@@ -149,7 +147,7 @@ public class EsQueryVistor implements QueryVisitor {
public void visit( WithinOperand op ) {
String name = op.getProperty().getValue();
- name = EsEntityIndexImpl.Types.LOCATION.PREFIX+ name.toLowerCase() ;
+ name = EsEntityIndexImpl.Types.LOCATION.PREFIX + name.toLowerCase() ;
float lat = op.getLatitude().getFloatValue();
float lon = op.getLongitude().getFloatValue();
@@ -162,30 +160,28 @@ public class EsQueryVistor implements QueryVisitor {
@Override
public void visit( LessThan op ) throws NoIndexException {
- String name = op.getProperty().getValue();
- name = name.toLowerCase();
+ String name = op.getProperty().getValue().toLowerCase();
Object value = op.getLiteral().getValue();
- if ( value instanceof String ) {
- name = addAnayzedSuffix( name );
- }
+
+ name = normalizeFieldName( name, value );
+
stack.push( QueryBuilders.rangeQuery( name ).lt( value ));
}
@Override
public void visit( LessThanEqual op ) throws NoIndexException {
- String name = op.getProperty().getValue();
- name = name.toLowerCase();
+ String name = op.getProperty().getValue().toLowerCase();
Object value = op.getLiteral().getValue();
- if ( value instanceof String ) {
- name = addAnayzedSuffix( name );
- }
+
+ name = normalizeFieldName( name, value );
+
+
stack.push( QueryBuilders.rangeQuery( name ).lte( value ));
}
@Override
public void visit( Equal op ) throws NoIndexException {
- String name = op.getProperty().getValue();
- name = name.toLowerCase();
+ String name = op.getProperty().getValue().toLowerCase();
Object value = op.getLiteral().getValue();
if ( value instanceof String ) {
@@ -193,45 +189,65 @@ public class EsQueryVistor implements QueryVisitor {
if ( svalue.indexOf("*") != -1 ) {
// for wildcard expression we need analuzed field, add suffix
- //name = addAnayzedSuffix( name );
+ name = EsEntityIndexImpl.Types.STRING_ANALYZED + name ;
stack.push( QueryBuilders.wildcardQuery(name, svalue) );
return;
}
// for equal operation on string, need to use unanalyzed field, leave off the suffix
- value = svalue;
- }
+ value = svalue;
+
+
+ }
+
+ name = normalizeFieldName( name, value );
+
stack.push( QueryBuilders.termQuery( name, value ));
}
@Override
public void visit( GreaterThan op ) throws NoIndexException {
- String name = op.getProperty().getValue();
- name = name.toLowerCase();
+ String name = op.getProperty().getValue().toLowerCase();
Object value = op.getLiteral().getValue();
- if ( value instanceof String ) {
- name = addAnayzedSuffix( name );
- }
+
+ name = normalizeFieldName( name, value );
+
stack.push( QueryBuilders.rangeQuery( name ).gt( value ) );
}
@Override
public void visit( GreaterThanEqual op ) throws NoIndexException {
- String name = op.getProperty().getValue();
- name = name.toLowerCase();
+ String name = op.getProperty().getValue().toLowerCase();
Object value = op.getLiteral().getValue();
- if ( value instanceof String ) {
- name = addAnayzedSuffix( name );
- }
+
+ name = normalizeFieldName( name, value );
+
stack.push( QueryBuilders.rangeQuery( name ).gte( value ) );
}
- private String addAnayzedSuffix( String name ) {
- if ( name.endsWith(EsEntityIndexImpl.ANALYZED_PREFIX) ) {
- return name;
- }
- return name + EsEntityIndexImpl.ANALYZED_PREFIX;
- }
+// private String addAnayzedSuffix( String name ) {
+// if ( name.endsWith(EsEntityIndexImpl.ANALYZED_PREFIX) ) {
+// return name;
+// }
+// return name + EsEntityIndexImpl.ANALYZED_PREFIX;
+// }
+
+
+ /**
+ * Normalizes a field based on it's search value. If it's a string, the string prefix is attached.
+ * If it's not, it's assumed to be a primitive
+ *
+ * @param fieldName
+ * @param value
+ * @return
+ */
+ private String normalizeFieldName(final String fieldName, final Object value){
+ if(value instanceof String){
+ return EsEntityIndexImpl.Types.STRING.PREFIX + fieldName;
+ }
+
+ return EsEntityIndexImpl.Types.PRIMITIVE.PREFIX + fieldName;
+ }
@Override
public QueryBuilder getQueryBuilder() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c66c864/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexMapUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexMapUtils.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexMapUtils.java
index 4f5ab88..ac29bae 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexMapUtils.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexMapUtils.java
@@ -51,7 +51,6 @@ import org.apache.usergrid.persistence.model.field.value.Location;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import static org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.ANALYZED_PREFIX;
import static org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.GEO_SUFFIX;
[2/2] git commit: Merge branch 'two-dot-o-candidate-serialization' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
two-dot-o-candidate-serialization
Posted by to...@apache.org.
Merge branch 'two-dot-o-candidate-serialization' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-candidate-serialization
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/363375d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/363375d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/363375d9
Branch: refs/heads/two-dot-o-candidate-serialization
Commit: 363375d9ca0c5e6554386cf0c9d3590b7c61d2b8
Parents: 2c66c86 101ac4f
Author: Todd Nine <to...@apache.org>
Authored: Wed Sep 24 13:21:38 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Wed Sep 24 13:21:38 2014 -0600
----------------------------------------------------------------------
.../org/apache/usergrid/tools/ESIndexTools.java | 223 +++++++++++++++++++
1 file changed, 223 insertions(+)
----------------------------------------------------------------------