You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by dr...@apache.org on 2016/12/20 10:04:39 UTC
incubator-unomi git commit: UNOMI-70 : use folding + keyword mapping.
detect field type in order to choose usage of keyword for sort and aggregates
Repository: incubator-unomi
Updated Branches:
refs/heads/feature-UNOMI-70-ES5X 603346ad3 -> 8b5a44bb1
UNOMI-70 : use folding + keyword mapping. detect field type in order to choose usage of keyword for sort and aggregates
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/8b5a44bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/8b5a44bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/8b5a44bb
Branch: refs/heads/feature-UNOMI-70-ES5X
Commit: 8b5a44bb1d9bc0090a6e4d11587d02b542acc567
Parents: 603346a
Author: Thomas Draier <dr...@apache.org>
Authored: Tue Dec 20 11:04:32 2016 +0100
Committer: Thomas Draier <dr...@apache.org>
Committed: Tue Dec 20 11:04:32 2016 +0100
----------------------------------------------------------------------
.../META-INF/cxs/mappings/geonameEntry.json | 48 +--------------
.../META-INF/cxs/mappings/userList.json | 15 -----
.../ElasticSearchPersistenceServiceImpl.java | 64 ++++++++++++++------
.../META-INF/cxs/mappings/_default_.json | 22 +++++++
.../META-INF/cxs/mappings/campaign.json | 24 --------
.../META-INF/cxs/mappings/campaignevent.json | 24 --------
.../resources/META-INF/cxs/mappings/event.json | 18 ------
.../resources/META-INF/cxs/mappings/goal.json | 19 +-----
.../META-INF/cxs/mappings/personaSession.json | 20 +-----
.../META-INF/cxs/mappings/profile.json | 14 +----
.../META-INF/cxs/mappings/propertyType.json | 45 --------------
.../resources/META-INF/cxs/mappings/rule.json | 12 ----
.../META-INF/cxs/mappings/scoring.json | 16 +----
.../META-INF/cxs/mappings/segment.json | 17 +-----
.../META-INF/cxs/mappings/session.json | 25 +-------
.../persistence/spi/PersistenceService.java | 10 +++
.../PropertyConditionESQueryBuilder.java | 4 +-
.../conditions/PropertyConditionEvaluator.java | 12 +++-
.../services/services/GoalsServiceImpl.java | 20 ++++--
.../services/services/QueryServiceImpl.java | 9 ++-
20 files changed, 124 insertions(+), 314 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json
----------------------------------------------------------------------
diff --git a/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json b/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json
index 612f3ca..64038ab 100644
--- a/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json
+++ b/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json
@@ -1,48 +1,10 @@
{
"geonameEntry": {
"properties": {
- "admin1Code": {
- "type": "keyword"
- },
- "admin2Code": {
- "type": "keyword"
- },
- "admin3Code": {
- "type": "keyword"
- },
- "admin4Code": {
- "type": "keyword"
- },
- "alternatenames": {
- "type": "keyword"
- },
- "asciiname": {
- "type": "keyword"
- },
- "cc2": {
- "type": "keyword"
- },
- "countryCode": {
- "type": "keyword"
- },
- "dem": {
- "type": "keyword"
- },
"elevation": {
"type": "long"
},
- "featureClass": {
- "type": "keyword"
- },
- "featureCode": {
- "type": "keyword"
- },
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
+
"location": {
"type": "geo_point"
},
@@ -50,15 +12,11 @@
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
- "name": {
- "type": "keyword"
- },
+
"population": {
"type": "long"
- },
- "timezone": {
- "type": "keyword"
}
+
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json
----------------------------------------------------------------------
diff --git a/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json b/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json
index 53fd043..d73e8b3 100644
--- a/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json
+++ b/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json
@@ -1,12 +1,6 @@
{
"userList": {
"properties": {
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
"metadata": {
"properties": {
"enabled": {
@@ -15,20 +9,11 @@
"hidden": {
"type": "boolean"
},
- "id": {
- "type": "keyword"
- },
"missingPlugins": {
"type": "boolean"
},
"readOnly": {
"type": "boolean"
- },
- "scope": {
- "type": "keyword"
- },
- "tags": {
- "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 5e806d1..ee85d98 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -33,10 +33,7 @@ import org.apache.unomi.api.query.DateRange;
import org.apache.unomi.api.query.IpRange;
import org.apache.unomi.api.query.NumericRange;
import org.apache.unomi.api.services.ClusterService;
-import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder;
-import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher;
-import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator;
-import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorDispatcher;
+import org.apache.unomi.persistence.elasticsearch.conditions.*;
import org.apache.unomi.persistence.spi.CustomObjectMapper;
import org.apache.unomi.persistence.spi.PersistenceService;
import org.apache.unomi.persistence.spi.aggregate.*;
@@ -175,6 +172,8 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private String bulkProcessorFlushInterval = "5s";
private String bulkProcessorBackoffPolicy = "exponential";
+ private Map<String, Map<String, Map<String, Object>>> knownMappings = new HashMap<>();
+
public void setBundleContext(BundleContext bundleContext) {
this.bundleContext = bundleContext;
}
@@ -388,6 +387,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
if (!indexExists) {
logger.info("{} index doesn't exist yet, creating it...", indexName);
Map<String,String> indexMappings = new HashMap<String,String>();
+ indexMappings.put("_default_",mappings.get("_default_"));
for (Map.Entry<String, String> entry : mappings.entrySet()) {
if (!itemsMonthlyIndexed.contains(entry.getKey()) && !indexNames.containsKey(entry.getKey())) {
indexMappings.put(entry.getKey(), entry.getValue());
@@ -631,6 +631,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
logger.info("{} index doesn't exist yet, creating it...", monthlyIndexName);
Map<String,String> indexMappings = new HashMap<String,String>();
+ indexMappings.put("_default_",mappings.get("_default_"));
for (Map.Entry<String, String> entry : mappings.entrySet()) {
if (itemsMonthlyIndexed.contains(entry.getKey())) {
indexMappings.put(entry.getKey(), entry.getValue());
@@ -928,6 +929,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
boolean indexExists = indicesExistsResponse.isExists();
if (!indexExists) {
Map<String,String> indexMappings = new HashMap<String,String>();
+ indexMappings.put("_default_",mappings.get("_default_"));
for (Map.Entry<String, String> entry : mappings.entrySet()) {
if (indexNames.containsKey(entry.getKey()) && indexNames.get(entry.getKey()).equals(indexName)) {
indexMappings.put(entry.getKey(), entry.getValue());
@@ -958,17 +960,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
CreateIndexRequestBuilder builder = client.admin().indices().prepareCreate(indexName)
.setSettings("{\n" +
" \"analysis\": {\n" +
- " \"tokenizer\": {\n" +
- " \"myTokenizer\": {\n" +
- " \"type\":\"pattern\",\n" +
- " \"pattern\":\".*\",\n" +
- " \"group\":0\n" +
- " }\n" +
- " },\n" +
" \"analyzer\": {\n" +
" \"folding\": {\n" +
" \"type\":\"custom\",\n" +
- " \"tokenizer\": \"myTokenizer\",\n" +
+ " \"tokenizer\": \"keyword\",\n" +
" \"filter\": [ \"lowercase\", \"asciifolding\" ]\n" +
" }\n" +
" }\n" +
@@ -995,6 +990,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
@Override
public void createMapping(String type, String source) {
+ if (type.equals("_default_")) {
+ return;
+ }
if (itemsMonthlyIndexed.contains(type)) {
createMapping(type, source, indexName + "-*");
} else if (indexNames.containsKey(type)) {
@@ -1042,6 +1040,36 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}.executeInClassLoader();
}
+ public Map<String, Object> getPropertyMapping(String property, String itemType) {
+ Map<String, Map<String, Object>> mappings = knownMappings.get(itemType);
+ Map<String, Object> result = getPropertyMapping(property, mappings);
+ if (result == null) {
+ mappings = getPropertiesMapping(itemType);
+ knownMappings.put(itemType, mappings);
+ result = getPropertyMapping(property, mappings);
+ }
+ return result;
+ }
+
+ private Map<String, Object> getPropertyMapping(String property, Map<String, Map<String, Object>> mappings) {
+ Map<String, Object> propMapping = null;
+ String[] properties = StringUtils.split(property, '.');
+ for (int i = 0; i < properties.length; i++) {
+ String s = properties[i];
+ if (mappings != null) {
+ propMapping = mappings.get(s);
+ if (i == properties.length - 1) {
+ return propMapping;
+ } else {
+ mappings = (Map<String, Map<String, Object>>) propMapping.get("properties");
+ }
+ } else {
+ return null;
+ }
+ }
+ return propMapping;
+ }
+
public boolean saveQuery(final String queryName, final String query) {
return new InClassLoaderExecute<Boolean>() {
protected Boolean execute(Object... args) {
@@ -1138,12 +1166,12 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
@Override
public <T extends Item> List<T> query(final String fieldName, final String[] fieldValues, String sortBy, final Class<T> clazz) {
- return query(QueryBuilders.termsQuery(fieldName, fieldValues), sortBy, clazz, 0, -1, getRouting(fieldName, fieldValues, clazz), null).getList();
+ return query(QueryBuilders.termsQuery(fieldName, ConditionContextHelper.foldToASCII(fieldValues)), sortBy, clazz, 0, -1, getRouting(fieldName, fieldValues, clazz), null).getList();
}
@Override
public <T extends Item> PartialList<T> query(String fieldName, String fieldValue, String sortBy, Class<T> clazz, int offset, int size) {
- return query(QueryBuilders.termQuery(fieldName, fieldValue), sortBy, clazz, offset, size, getRouting(fieldName, new String[]{fieldValue}, clazz), null);
+ return query(QueryBuilders.termQuery(fieldName, ConditionContextHelper.foldToASCII(fieldValue)), sortBy, clazz, offset, size, getRouting(fieldName, new String[]{fieldValue}, clazz), null);
}
@Override
@@ -1247,9 +1275,11 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
} else {
String name = StringUtils.substringBeforeLast(sortByElement,":");
- if (name.equals("metadata.name") || name.equals("metadata.description")
- || name.equals("properties.firstName") || name.equals("properties.lastName")
- ) {
+ Map<String,Object> propertyMapping = getPropertyMapping(name,itemType);
+ if (propertyMapping != null
+ && "text".equals(propertyMapping.get("type"))
+ && propertyMapping.containsKey("fields")
+ && ((Map)propertyMapping.get("fields")).containsKey("keyword")) {
name += ".keyword";
}
if (sortByElement.endsWith(":desc")) {
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json
new file mode 100644
index 0000000..9bc4ab0
--- /dev/null
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json
@@ -0,0 +1,22 @@
+{
+ "_default_": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
index acd4b85..3e36a7f 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
@@ -4,27 +4,12 @@
"cost": {
"type": "double"
},
- "currency": {
- "type": "keyword"
- },
"startDate": {
"type": "date"
},
"endDate": {
"type": "date"
},
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
- "primaryGoal": {
- "type": "keyword"
- },
- "timezone": {
- "type": "keyword"
- },
"metadata": {
"properties": {
"enabled": {
@@ -33,20 +18,11 @@
"hidden": {
"type": "boolean"
},
- "id": {
- "type": "keyword"
- },
"missingPlugins": {
"type": "boolean"
},
"readOnly": {
"type": "boolean"
- },
- "scope": {
- "type": "keyword"
- },
- "tags": {
- "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
index 7308995..9ab8779 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
@@ -1,27 +1,12 @@
{
"campaignevent": {
"properties": {
- "campaignId": {
- "type": "keyword"
- },
"cost": {
"type": "double"
},
- "currency": {
- "type": "keyword"
- },
"eventDate": {
"type": "date"
},
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
- "timezone": {
- "type": "keyword"
- },
"metadata": {
"properties": {
"enabled": {
@@ -30,20 +15,11 @@
"hidden": {
"type": "boolean"
},
- "id": {
- "type": "keyword"
- },
"missingPlugins": {
"type": "boolean"
},
"readOnly": {
"type": "boolean"
- },
- "scope": {
- "type": "keyword"
- },
- "tags": {
- "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
index c240b3b..69c9f08 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
@@ -1,24 +1,6 @@
{
"event": {
"properties": {
- "eventType": {
- "type": "keyword"
- },
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
- "profileId": {
- "type": "keyword"
- },
- "scope": {
- "type": "keyword"
- },
- "sessionId": {
- "type": "keyword"
- },
"timeStamp": {
"type": "date"
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
index 081915b..3e0bb6f 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
@@ -1,15 +1,7 @@
{
"goal": {
"properties": {
- "campaignId": {
- "type": "keyword"
- },
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
+
"metadata": {
"properties": {
"enabled": {
@@ -18,20 +10,11 @@
"hidden": {
"type": "boolean"
},
- "id": {
- "type": "keyword"
- },
"missingPlugins": {
"type": "boolean"
},
"readOnly": {
"type": "boolean"
- },
- "scope": {
- "type": "keyword"
- },
- "tags": {
- "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
index cb26f1e..ce3a8f3 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
@@ -4,31 +4,13 @@
"duration" : {
"type" : "long"
},
- "itemId" : {
- "type" : "keyword"
- },
- "itemType" : {
- "type" : "keyword"
- },
+
"timeStamp": {
"type": "date"
},
"lastEventDate": {
"type": "date"
},
- "profile" : {
- "properties" : {
- "itemId" : {
- "type" : "keyword"
- },
- "itemType" : {
- "type" : "keyword"
- }
- }
- },
- "profileId" : {
- "type" : "keyword"
- },
"properties" : {
"properties" : {
"location" : {
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
index 665941d..29d40b7 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
@@ -1,12 +1,7 @@
{
"profile": {
"properties": {
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
+
"properties": {
"properties": {
"firstVisit": {
@@ -22,13 +17,6 @@
"type": "long"
}
}
- },
- "systemProperties" : {
- "properties" : {
- "mergeIdentifier" : {
- "type" : "keyword"
- }
- }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
index bdba719..99969f7 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
@@ -1,34 +1,10 @@
{
"propertyType": {
"properties": {
- "automaticMappingsFrom": {
- "type": "keyword"
- },
"dateRanges": {
"properties": {
- "from": {
- "type": "keyword"
- },
- "key": {
- "type": "keyword"
- },
- "to": {
- "type": "keyword"
- }
}
},
- "defaultValue": {
- "type": "keyword"
- },
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
- "mergeStrategy": {
- "type": "keyword"
- },
"metadata": {
"properties": {
"enabled": {
@@ -37,20 +13,11 @@
"hidden": {
"type": "boolean"
},
- "id": {
- "type": "keyword"
- },
"missingPlugins": {
"type": "boolean"
},
"readOnly": {
"type": "boolean"
- },
- "scope": {
- "type": "keyword"
- },
- "tags": {
- "type": "keyword"
}
}
},
@@ -62,9 +29,6 @@
"from": {
"type": "double"
},
- "key": {
- "type": "keyword"
- },
"to": {
"type": "double"
}
@@ -75,15 +39,6 @@
},
"rank": {
"type": "double"
- },
- "tags": {
- "type": "keyword"
- },
- "target": {
- "type": "keyword"
- },
- "type": {
- "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
index aad6029..ea6bb26 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
@@ -1,12 +1,6 @@
{
"rule": {
"properties": {
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
"metadata": {
"properties": {
"enabled": {
@@ -15,17 +9,11 @@
"hidden": {
"type": "boolean"
},
- "id": {
- "type": "keyword"
- },
"missingPlugins": {
"type": "boolean"
},
"readOnly": {
"type": "boolean"
- },
- "scope": {
- "type": "keyword"
}
}
},
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
index b46d7e2..cda229b 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
@@ -1,12 +1,7 @@
{
"scoring": {
"properties": {
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
+
"metadata": {
"properties": {
"enabled": {
@@ -15,20 +10,11 @@
"hidden": {
"type": "boolean"
},
- "id": {
- "type": "keyword"
- },
"missingPlugins": {
"type": "boolean"
},
"readOnly": {
"type": "boolean"
- },
- "scope": {
- "type": "keyword"
- },
- "tags": {
- "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
index 322ff19..5e20436 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
@@ -1,12 +1,7 @@
{
"segment": {
"properties": {
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
+
"metadata": {
"properties": {
"enabled": {
@@ -15,20 +10,12 @@
"hidden": {
"type": "boolean"
},
- "id": {
- "type": "keyword"
- },
+
"missingPlugins": {
"type": "boolean"
},
"readOnly": {
"type": "boolean"
- },
- "scope": {
- "type": "keyword"
- },
- "tags": {
- "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
index b1759a3..d687dce 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
@@ -4,28 +4,11 @@
"duration": {
"type": "long"
},
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
+
"lastEventDate": {
"type": "date"
},
- "profileId": {
- "type": "keyword"
- },
- "profile": {
- "properties": {
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- }
- }
- },
+
"properties": {
"properties": {
"location": {
@@ -33,9 +16,7 @@
}
}
},
- "scope": {
- "type": "keyword"
- },
+
"size": {
"type": "long"
},
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
----------------------------------------------------------------------
diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
index e191829..132c583 100644
--- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
+++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
@@ -174,6 +174,16 @@ public interface PersistenceService {
Map<String, Map<String, Object>> getPropertiesMapping(String itemType);
/**
+ * Retrieve the mapping for one specific property for a given type.
+ *
+ * @param property the property name (can use nested dot notation)
+ * @param itemType the itemType we want to retrieve the mappings for
+ * @return
+ */
+ Map<String, Object> getPropertyMapping(String property, String itemType);
+
+
+ /**
* Create mapping
* @param type
* @param source
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
index 31aa0a3..c36722b 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
@@ -44,12 +44,12 @@ public class PropertyConditionESQueryBuilder implements ConditionESQueryBuilder
throw new IllegalArgumentException("Impossible to build ES filter, condition is not valid, comparisonOperator and propertyName properties should be provided");
}
- String expectedValue = (String) condition.getParameter("propertyValue");
+ String expectedValue = ConditionContextHelper.foldToASCII((String) condition.getParameter("propertyValue"));
Object expectedValueInteger = condition.getParameter("propertyValueInteger");
Object expectedValueDate = condition.getParameter("propertyValueDate");
Object expectedValueDateExpr = condition.getParameter("propertyValueDateExpr");
- List<?> expectedValues = (List<?>) condition.getParameter("propertyValues");
+ List<?> expectedValues = ConditionContextHelper.foldToASCII((List<?>) condition.getParameter("propertyValues"));
List<?> expectedValuesInteger = (List<?>) condition.getParameter("propertyValuesInteger");
List<?> expectedValuesDate = (List<?>) condition.getParameter("propertyValuesDate");
List<?> expectedValuesDateExpr = (List<?>) condition.getParameter("propertyValuesDateExpr");
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
index d8b7754..68e58a6 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
@@ -83,7 +83,7 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
return false;
}
- List<Object> actual = getValueSet(actualValue);
+ List<Object> actual = ConditionContextHelper.foldToASCII(getValueSet(actualValue));
boolean result = true;
@@ -136,7 +136,7 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
String op = (String) condition.getParameter("comparisonOperator");
String name = (String) condition.getParameter("propertyName");
- String expectedValue = (String) condition.getParameter("propertyValue");
+ String expectedValue = ConditionContextHelper.foldToASCII((String) condition.getParameter("propertyValue"));
Object expectedValueInteger = condition.getParameter("propertyValueInteger");
Object expectedValueDate = condition.getParameter("propertyValueDate");
Object expectedValueDateExpr = condition.getParameter("propertyValueDateExpr");
@@ -165,6 +165,9 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
actualValue = null;
}
}
+ if (actualValue instanceof String) {
+ actualValue = ConditionContextHelper.foldToASCII((String) actualValue);
+ }
if(op == null) {
return false;
@@ -175,6 +178,9 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
} else if (op.equals("equals")) {
if (actualValue instanceof Collection) {
for (Object o : ((Collection<?>)actualValue)) {
+ if (o instanceof String) {
+ o = ConditionContextHelper.foldToASCII((String) o);
+ }
if (compare(o, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr) == 0) {
return true;
}
@@ -214,7 +220,7 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
} else if (op.equals("matchesRegex")) {
return expectedValue != null && Pattern.compile(expectedValue).matcher(actualValue.toString()).matches();
} else if (op.equals("in") || op.equals("notIn") || op.equals("hasSomeOf") || op.equals("hasNoneOf") || op.equals("all")) {
- List<?> expectedValues = (List<?>) condition.getParameter("propertyValues");
+ List<?> expectedValues = ConditionContextHelper.foldToASCII((List<?>) condition.getParameter("propertyValues"));
List<?> expectedValuesInteger = (List<?>) condition.getParameter("propertyValuesInteger");
List<?> expectedValuesDate = (List<?>) condition.getParameter("propertyValuesDate");
List<?> expectedValuesDateExpr = (List<?>) condition.getParameter("propertyValuesDateExpr");
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java
----------------------------------------------------------------------
diff --git a/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java
index a99f155..a6bd1d6 100644
--- a/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java
@@ -474,25 +474,33 @@ public class GoalsServiceImpl implements GoalsService, SynchronousBundleListener
// resolve aggregate
BaseAggregate aggregate = null;
- if(query != null && query.getAggregate() != null && query.getAggregate().getProperty() != null) {
+ String property = query.getAggregate().getProperty();
+ if(query != null && query.getAggregate() != null && property != null) {
if (query.getAggregate().getType() != null){
// try to guess the aggregate type
if(query.getAggregate().getType().equals("date")) {
String interval = (String) query.getAggregate().getParameters().get("interval");
String format = (String) query.getAggregate().getParameters().get("format");
- aggregate = new DateAggregate(query.getAggregate().getProperty(), interval, format);
+ aggregate = new DateAggregate(property, interval, format);
} else if (query.getAggregate().getType().equals("dateRange") && query.getAggregate().getDateRanges() != null && query.getAggregate().getDateRanges().size() > 0) {
String format = (String) query.getAggregate().getParameters().get("format");
- aggregate = new DateRangeAggregate(query.getAggregate().getProperty(), format, query.getAggregate().getDateRanges());
+ aggregate = new DateRangeAggregate(property, format, query.getAggregate().getDateRanges());
} else if (query.getAggregate().getType().equals("numericRange") && query.getAggregate().getNumericRanges() != null && query.getAggregate().getNumericRanges().size() > 0) {
- aggregate = new NumericRangeAggregate(query.getAggregate().getProperty(), query.getAggregate().getNumericRanges());
+ aggregate = new NumericRangeAggregate(property, query.getAggregate().getNumericRanges());
} else if (query.getAggregate().getType().equals("ipRange") && query.getAggregate().ipRanges() != null && query.getAggregate().ipRanges().size() > 0) {
- aggregate = new IpRangeAggregate(query.getAggregate().getProperty(), query.getAggregate().ipRanges());
+ aggregate = new IpRangeAggregate(property, query.getAggregate().ipRanges());
}
}
if(aggregate == null){
- aggregate = new TermsAggregate(query.getAggregate().getProperty() + ".keyword");
+ Map<String,Object> propertyMapping = persistenceService.getPropertyMapping(property, Session.ITEM_TYPE);
+ if (propertyMapping != null
+ && "text".equals(propertyMapping.get("type"))
+ && propertyMapping.containsKey("fields")
+ && ((Map)propertyMapping.get("fields")).containsKey("keyword")) {
+ property += ".keyword";
+ }
+ aggregate = new TermsAggregate(property);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
----------------------------------------------------------------------
diff --git a/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
index 79edd43..30e6dab 100644
--- a/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
@@ -83,7 +83,14 @@ public class QueryServiceImpl implements QueryService {
}
// fall back on terms aggregate
- return persistenceService.aggregateQuery(query.getCondition(), new TermsAggregate(property + ".keyword"), itemType);
+ Map<String,Object> propertyMapping = persistenceService.getPropertyMapping( property, itemType);
+ if (propertyMapping != null
+ && "text".equals(propertyMapping.get("type"))
+ && propertyMapping.containsKey("fields")
+ && ((Map)propertyMapping.get("fields")).containsKey("keyword")) {
+ property += ".keyword";
+ }
+ return persistenceService.aggregateQuery(query.getCondition(), new TermsAggregate(property), itemType);
}
return getAggregate(itemType, property);