You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by sh...@apache.org on 2017/01/09 16:21:35 UTC
[01/50] [abbrv] incubator-unomi git commit: UNOMI-63 Use
ElasticSearch BulkProcessing to perform segment updates - Validate
configuration functionality
Repository: incubator-unomi
Updated Branches:
refs/heads/master a4955cafa -> b1c88c4c4
UNOMI-63 Use ElasticSearch BulkProcessing to perform segment updates
- Validate configuration functionality
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/a7fb48dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/a7fb48dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/a7fb48dd
Branch: refs/heads/master
Commit: a7fb48dd948d36e376a4fdfc8868fd51a9053bd0
Parents: 79b1b9c
Author: Serge Huber <sh...@apache.org>
Authored: Tue Nov 15 14:13:46 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Tue Nov 15 14:13:46 2016 +0100
----------------------------------------------------------------------
.../ElasticSearchPersistenceServiceImpl.java | 85 +++++++++++++-------
.../resources/OSGI-INF/blueprint/blueprint.xml | 13 +++
...g.apache.unomi.persistence.elasticsearch.cfg | 6 ++
3 files changed, 77 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a7fb48dd/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index d13f7b6..2208951 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -117,12 +117,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
public static final String CONTEXTSERVER_PORT = "contextserver.port";
public static final String CONTEXTSERVER_SECURE_ADDRESS = "contextserver.secureAddress";
public static final String CONTEXTSERVER_SECURE_PORT = "contextserver.securePort";
- public static final String CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_NAME = "contextserver.elasticsearch.bulkprocessor.name";
- public static final String CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_CONCURRENTREQUEST = "contextserver.elasticsearch.bulkprocessor.concurrentRequests";
- public static final String CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_BULKACTIONS = "contextserver.elasticsearch.bulkprocessor.bulkActions";
- public static final String CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_BULKSIZE = "contextserver.elasticsearch.bulkprocessor.bulkSize";
- public static final String CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_FLUSHINTERVAL = "contextserver.elasticsearch.bulkprocessor.flushInterval";
- public static final String CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_BACKOFFPOLICY = "contextserver.elasticsearch.bulkprocessor.backoffPolicy";
public static final String KARAF_HOME = "karaf.home";
public static final String ELASTICSEARCH_HOME_DIRECTORY = "elasticsearch";
public static final String ELASTICSEARCH_PLUGINS_DIRECTORY = ELASTICSEARCH_HOME_DIRECTORY + "/plugins";
@@ -142,6 +136,12 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
public static final String PATH_PLUGINS = "path.plugins";
public static final String INDEX_MAX_RESULT_WINDOW = "index.max_result_window";
public static final String MAPPER_ALLOW_DOTS_IN_NAME = "mapper.allow_dots_in_name";
+ public static final String BULK_PROCESSOR_NAME = "bulkProcessor.name";
+ public static final String BULK_PROCESSOR_CONCURRENT_REQUESTS = "bulkProcessor.concurrentRequests";
+ public static final String BULK_PROCESSOR_BULK_ACTIONS = "bulkProcessor.bulkActions";
+ public static final String BULK_PROCESSOR_BULK_SIZE = "bulkProcessor.bulkSize";
+ public static final String BULK_PROCESSOR_FLUSH_INTERVAL = "bulkProcessor.flushInterval";
+ public static final String BULK_PROCESSOR_BACKOFF_POLICY = "bulkProcessor.backoffPolicy";
private Node node;
private Client client;
@@ -172,6 +172,13 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private Timer timer;
+ private String bulkProcessorName = "unomi-bulk";
+ private String bulkProcessorConcurrentRequests = "1";
+ private String bulkProcessorBulkActions = "1000";
+ private String bulkProcessorBulkSize= "5MB";
+ private String bulkProcessorFlushInterval = "5s";
+ private String bulkProcessorBackoffPolicy = "exponential";
+
public void setBundleContext(BundleContext bundleContext) {
this.bundleContext = bundleContext;
}
@@ -252,6 +259,30 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
this.conditionESQueryBuilderDispatcher = conditionESQueryBuilderDispatcher;
}
+ public void setBulkProcessorName(String bulkProcessorName) {
+ this.bulkProcessorName = bulkProcessorName;
+ }
+
+ public void setBulkProcessorConcurrentRequests(String bulkProcessorConcurrentRequests) {
+ this.bulkProcessorConcurrentRequests = bulkProcessorConcurrentRequests;
+ }
+
+ public void setBulkProcessorBulkActions(String bulkProcessorBulkActions) {
+ this.bulkProcessorBulkActions = bulkProcessorBulkActions;
+ }
+
+ public void setBulkProcessorBulkSize(String bulkProcessorBulkSize) {
+ this.bulkProcessorBulkSize = bulkProcessorBulkSize;
+ }
+
+ public void setBulkProcessorFlushInterval(String bulkProcessorFlushInterval) {
+ this.bulkProcessorFlushInterval = bulkProcessorFlushInterval;
+ }
+
+ public void setBulkProcessorBackoffPolicy(String bulkProcessorBackoffPolicy) {
+ this.bulkProcessorBackoffPolicy = bulkProcessorBackoffPolicy;
+ }
+
public void start() {
loadPredefinedMappings(bundleContext, false);
@@ -282,6 +313,13 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
secureAddress = System.getProperty(CONTEXTSERVER_SECURE_ADDRESS, secureAddress);
securePort = System.getProperty(CONTEXTSERVER_SECURE_PORT, securePort);
+ bulkProcessorName = System.getProperty(BULK_PROCESSOR_NAME, bulkProcessorName);
+ bulkProcessorConcurrentRequests = System.getProperty(BULK_PROCESSOR_CONCURRENT_REQUESTS, bulkProcessorConcurrentRequests);
+ bulkProcessorBulkActions = System.getProperty(BULK_PROCESSOR_BULK_ACTIONS, bulkProcessorBulkActions);
+ bulkProcessorBulkSize = System.getProperty(BULK_PROCESSOR_BULK_SIZE, bulkProcessorBulkSize);
+ bulkProcessorFlushInterval = System.getProperty(BULK_PROCESSOR_FLUSH_INTERVAL, bulkProcessorFlushInterval);
+ bulkProcessorBackoffPolicy = System.getProperty(BULK_PROCESSOR_BACKOFF_POLICY, bulkProcessorBackoffPolicy);
+
Settings.Builder settingsBuilder = Settings.builder();
if (settings != null) {
settingsBuilder.put(settings);
@@ -430,37 +468,30 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
// we could add index creation here in the case of index seperation by dates.
}
});
- if (System.getProperty(CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_NAME) != null) {
- String name = System.getProperty(CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_NAME);
- if (name != null && name.length() > 0) {
- bulkProcessorBuilder.setName(name);
- }
+ if (bulkProcessorName != null && bulkProcessorName.length() > 0) {
+ bulkProcessorBuilder.setName(bulkProcessorName);
}
- if (System.getProperty(CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_CONCURRENTREQUEST) != null) {
- String concurrentRequestsStr = System.getProperty(CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_CONCURRENTREQUEST);
- int concurrentRequests = Integer.parseInt(concurrentRequestsStr);
+ if (bulkProcessorConcurrentRequests != null) {
+ int concurrentRequests = Integer.parseInt(bulkProcessorConcurrentRequests);
if (concurrentRequests > 1) {
bulkProcessorBuilder.setConcurrentRequests(concurrentRequests);
}
}
- if (System.getProperty(CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_BULKACTIONS) != null) {
- String bulkActionsStr = System.getProperty(CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_BULKACTIONS);
- int bulkActions = Integer.parseInt(bulkActionsStr);
+ if (bulkProcessorBulkActions != null) {
+ int bulkActions = Integer.parseInt(bulkProcessorBulkActions);
bulkProcessorBuilder.setBulkActions(bulkActions);
}
- if (System.getProperty(CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_BULKSIZE) != null) {
- String bulkSizeStr = System.getProperty(CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_BULKSIZE);
- bulkProcessorBuilder.setBulkSize(ByteSizeValue.parseBytesSizeValue(bulkSizeStr, new ByteSizeValue(5, ByteSizeUnit.MB), CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_BULKSIZE));
+ if (bulkProcessorBulkSize != null) {
+ bulkProcessorBuilder.setBulkSize(ByteSizeValue.parseBytesSizeValue(bulkProcessorBulkSize, new ByteSizeValue(5, ByteSizeUnit.MB), BULK_PROCESSOR_BULK_SIZE));
}
- if (System.getProperty(CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_FLUSHINTERVAL) != null) {
- String flushIntervalStr = System.getProperty(CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_FLUSHINTERVAL);
- bulkProcessorBuilder.setFlushInterval(TimeValue.parseTimeValue(flushIntervalStr, null, CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_FLUSHINTERVAL));
+ if (bulkProcessorFlushInterval != null) {
+ bulkProcessorBuilder.setFlushInterval(TimeValue.parseTimeValue(bulkProcessorFlushInterval, null, BULK_PROCESSOR_FLUSH_INTERVAL));
} else {
// in ElasticSearch this defaults to null, but we would like to set a value to 5 seconds by default
bulkProcessorBuilder.setFlushInterval(new TimeValue(5, TimeUnit.SECONDS));
}
- if (System.getProperty(CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_BACKOFFPOLICY) != null) {
- String backoffPolicyStr = System.getProperty(CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_BACKOFFPOLICY);
+ if (bulkProcessorBackoffPolicy != null) {
+ String backoffPolicyStr = bulkProcessorBackoffPolicy;
if (backoffPolicyStr != null && backoffPolicyStr.length() > 0) {
backoffPolicyStr = backoffPolicyStr.toLowerCase();
if ("nobackoff".equals(backoffPolicyStr)) {
@@ -469,7 +500,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
int paramStartPos = backoffPolicyStr.indexOf("constant(" + "constant(".length());
int paramEndPos = backoffPolicyStr.indexOf(")", paramStartPos);
int paramSeparatorPos = backoffPolicyStr.indexOf(",", paramStartPos);
- TimeValue delay = TimeValue.parseTimeValue(backoffPolicyStr.substring(paramStartPos, paramSeparatorPos), new TimeValue(5, TimeUnit.SECONDS), CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_BACKOFFPOLICY );
+ TimeValue delay = TimeValue.parseTimeValue(backoffPolicyStr.substring(paramStartPos, paramSeparatorPos), new TimeValue(5, TimeUnit.SECONDS), BULK_PROCESSOR_BACKOFF_POLICY);
int maxNumberOfRetries = Integer.parseInt(backoffPolicyStr.substring(paramSeparatorPos+1, paramEndPos));
bulkProcessorBuilder.setBackoffPolicy(BackoffPolicy.constantBackoff(delay, maxNumberOfRetries));
} else if (backoffPolicyStr.startsWith("exponential")) {
@@ -480,7 +511,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
int paramStartPos = backoffPolicyStr.indexOf("exponential(" + "exponential(".length());
int paramEndPos = backoffPolicyStr.indexOf(")", paramStartPos);
int paramSeparatorPos = backoffPolicyStr.indexOf(",", paramStartPos);
- TimeValue delay = TimeValue.parseTimeValue(backoffPolicyStr.substring(paramStartPos, paramSeparatorPos), new TimeValue(5, TimeUnit.SECONDS), CONTEXTSERVER_ELASTICSEARCH_BULKPROCESSOR_BACKOFFPOLICY );
+ TimeValue delay = TimeValue.parseTimeValue(backoffPolicyStr.substring(paramStartPos, paramSeparatorPos), new TimeValue(5, TimeUnit.SECONDS), BULK_PROCESSOR_BACKOFF_POLICY);
int maxNumberOfRetries = Integer.parseInt(backoffPolicyStr.substring(paramSeparatorPos+1, paramEndPos));
bulkProcessorBuilder.setBackoffPolicy(BackoffPolicy.exponentialBackoff(delay, maxNumberOfRetries));
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a7fb48dd/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index 5fbc9dd..75039d8 100644
--- a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -40,6 +40,13 @@
<cm:property name="discovery.zen.ping.multicast.enabled" value="false"/>
<cm:property name="node.data" value="true"/>
<cm:property name="defaultQueryLimit" value="10"/>
+
+ <cm:property name="bulkProcessor.name" value="unomi-bulk" />
+ <cm:property name="bulkProcessor.concurrentRequests" value="1" />
+ <cm:property name="bulkProcessor.bulkActions" value="1000" />
+ <cm:property name="bulkProcessor.bulkSize" value="5MB" />
+ <cm:property name="bulkProcessor.flushInterval" value="5s" />
+ <cm:property name="bulkProcessor.backoffPolicy" value="exponential" />
</cm:default-properties>
</cm:property-placeholder>
@@ -106,6 +113,12 @@
<entry key="geonameEntry" value="geonames"/>
</map>
</property>
+ <property name="bulkProcessorName" value="${es.bulkProcessor.name}" />
+ <property name="bulkProcessorConcurrentRequests" value="${es.bulkProcessor.concurrentRequests}" />
+ <property name="bulkProcessorBulkActions" value="${es.bulkProcessor.bulkActions}" />
+ <property name="bulkProcessorBulkSize" value="${es.bulkProcessor.bulkSize}" />
+ <property name="bulkProcessorFlushInterval" value="${es.bulkProcessor.flushInterval}" />
+ <property name="bulkProcessorBackoffPolicy" value="${es.bulkProcessor.backoffPolicy}" />
</bean>
</blueprint>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a7fb48dd/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
index fb058a4..c5d906f 100644
--- a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
+++ b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
@@ -26,3 +26,9 @@ node.data=true
discovery.zen.ping.multicast.enabled=false
#discovery.zen.ping.unicast.hosts=["192.168.0.1:9300", "192.168.0.2:9300"]
defaultQueryLimit=10
+bulkProcessor.name=unomi-bulk
+bulkProcessor.concurrentRequests=1
+bulkProcessor.bulkActions=1000
+bulkProcessor.bulkSize=10MB
+bulkProcessor.flushInterval=5s
+bulkProcessor.backoffPolicy=exponential
\ No newline at end of file
[44/50] [abbrv] incubator-unomi git commit: UNOMI-70 : restored
numberOfShards / numberOfReplicas
Posted by sh...@apache.org.
UNOMI-70 : restored numberOfShards / numberOfReplicas
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/f9418501
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/f9418501
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/f9418501
Branch: refs/heads/master
Commit: f9418501f47b6dbd577927f7fc53f69b377f9eca
Parents: 88f493c
Author: Thomas Draier <dr...@apache.org>
Authored: Tue Jan 3 19:01:41 2017 +0100
Committer: Thomas Draier <dr...@apache.org>
Committed: Tue Jan 3 19:01:41 2017 +0100
----------------------------------------------------------------------
.../ElasticSearchPersistenceServiceImpl.java | 14 ++++++++++++++
.../main/resources/OSGI-INF/blueprint/blueprint.xml | 4 ++++
2 files changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/f9418501/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index b3b569c..0958bcd 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -135,6 +135,8 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private String indexName;
private String monthlyIndexNumberOfShards;
private String monthlyIndexNumberOfReplicas;
+ private String numberOfShards;
+ private String numberOfReplicas;
private String elasticSearchConfig = null;
private BundleContext bundleContext;
private Map<String, String> mappings = new HashMap<String, String>();
@@ -191,6 +193,14 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
this.monthlyIndexNumberOfReplicas = monthlyIndexNumberOfReplicas;
}
+ public void setNumberOfShards(String numberOfShards) {
+ this.numberOfShards = numberOfShards;
+ }
+
+ public void setNumberOfReplicas(String numberOfReplicas) {
+ this.numberOfReplicas = numberOfReplicas;
+ }
+
public void setAddress(String address) {
this.address = address;
}
@@ -966,6 +976,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private void internalCreateIndex(String indexName, Map<String,String> mappings) {
CreateIndexRequestBuilder builder = client.admin().indices().prepareCreate(indexName)
.setSettings("{\n" +
+ " \"index\" : {\n" +
+ " \"number_of_shards\" : " + numberOfShards + ",\n" +
+ " \"number_of_replicas\" : " + numberOfReplicas + "\n" +
+ " },\n" +
" \"analysis\": {\n" +
" \"analyzer\": {\n" +
" \"folding\": {\n" +
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/f9418501/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index fea1879..c929009 100644
--- a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -33,6 +33,8 @@
<cm:property name="cluster.name" value="contextElasticSearch"/>
<cm:property name="index.name" value="context"/>
<cm:property name="elasticSearchConfig" value="file:${karaf.etc}/elasticsearch.yml"/>
+ <cm:property name="numberOfShards" value="5"/>
+ <cm:property name="numberOfReplicas" value="0"/>
<cm:property name="monthlyIndex.numberOfShards" value="3"/>
<cm:property name="monthlyIndex.numberOfReplicas" value="0"/>
<cm:property name="defaultQueryLimit" value="10"/>
@@ -92,6 +94,8 @@
<property name="indexName" value="${es.index.name}"/>
<property name="monthlyIndexNumberOfShards" value="${es.monthlyIndex.numberOfShards}"/>
<property name="monthlyIndexNumberOfReplicas" value="${es.monthlyIndex.numberOfReplicas}"/>
+ <property name="numberOfShards" value="${es.numberOfShards}"/>
+ <property name="numberOfReplicas" value="${es.numberOfReplicas}"/>
<property name="address" value="${web.contextserver.address}"/>
<property name="port" value="${web.contextserver.port}"/>
<property name="secureAddress" value="${web.contextserver.secureAddress}"/>
[10/50] [abbrv] incubator-unomi git commit: UNOMI-63 Use
ElasticSearch BulkProcessing to perform segment updates Replace lists with
sets to optimize porfile removal
Posted by sh...@apache.org.
UNOMI-63 Use ElasticSearch BulkProcessing to perform segment updates
Replace lists with sets to optimize porfile removal
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/85f00e8a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/85f00e8a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/85f00e8a
Branch: refs/heads/master
Commit: 85f00e8ace06eb603619d604718879ef39d29f20
Parents: 1075a02
Author: Serge Huber <sh...@apache.org>
Authored: Thu Nov 24 20:17:21 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Thu Nov 24 20:17:21 2016 +0100
----------------------------------------------------------------------
.../unomi/services/services/SegmentServiceImpl.java | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/85f00e8a/services/src/main/java/org/apache/unomi/services/services/SegmentServiceImpl.java
----------------------------------------------------------------------
diff --git a/services/src/main/java/org/apache/unomi/services/services/SegmentServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/SegmentServiceImpl.java
index 06b83b0..d1f120f 100644
--- a/services/src/main/java/org/apache/unomi/services/services/SegmentServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/services/SegmentServiceImpl.java
@@ -825,21 +825,25 @@ public class SegmentServiceImpl implements SegmentService, SynchronousBundleList
segmentCondition.setParameter("propertyValue", segment.getItemId());
if(segment.getMetadata().isEnabled()) {
+ // the following list can grow really big if the segments are large.
+ // We might want to replace this with scrolling if it becomes huge
+ // (100million profiles)
List<Profile> previousProfiles = persistenceService.query(segmentCondition, null, Profile.class);
List<Profile> newProfiles = persistenceService.query(segment.getCondition(), null, Profile.class);
- List<Profile> add = new ArrayList<>(newProfiles);
- add.removeAll(previousProfiles);
- previousProfiles.removeAll(newProfiles);
+ Set<Profile> profilesToAdd = new HashSet<>(newProfiles);
+ Set<Profile> profilesToRemove = new HashSet<>(previousProfiles);
+ profilesToAdd.removeAll(previousProfiles);
+ profilesToRemove.removeAll(newProfiles);
- for (Profile profileToAdd : add) {
+ for (Profile profileToAdd : profilesToAdd) {
profileToAdd.getSegments().add(segment.getItemId());
persistenceService.update(profileToAdd.getItemId(), null, Profile.class, "segments", profileToAdd.getSegments());
Event profileUpdated = new Event("profileUpdated", null, profileToAdd, null, null, profileToAdd, new Date());
profileUpdated.setPersistent(false);
eventService.send(profileUpdated);
}
- for (Profile profileToRemove : previousProfiles) {
+ for (Profile profileToRemove : profilesToRemove) {
profileToRemove.getSegments().remove(segment.getItemId());
persistenceService.update(profileToRemove.getItemId(), null, Profile.class, "segments", profileToRemove.getSegments());
Event profileUpdated = new Event("profileUpdated", null, profileToRemove, null, null, profileToRemove, new Date());
[37/50] [abbrv] incubator-unomi git commit: - Add logging to help
diagnose problems in the condition builders registration
Posted by sh...@apache.org.
- Add logging to help diagnose problems in the condition builders registration
Signed-off-by: Serge Huber <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/952da614
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/952da614
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/952da614
Branch: refs/heads/master
Commit: 952da6144726173d85827fbc5f838521033c061f
Parents: 8d54f8d
Author: Serge Huber <sh...@apache.org>
Authored: Wed Dec 21 21:27:40 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Wed Dec 21 21:27:40 2016 +0100
----------------------------------------------------------------------
.../elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/952da614/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java
index 5289f69..3ab0d5e 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java
@@ -97,6 +97,7 @@ public class ConditionESQueryBuilderDispatcher {
}
// if no matching
+ logger.warn("No matching query builder for condition {} and context {}", condition, context);
return QueryBuilders.matchAllQuery();
}
[49/50] [abbrv] incubator-unomi git commit:
[jgitflow-maven-plugin]updating poms for unomi_70_es_5_x version
Posted by sh...@apache.org.
[jgitflow-maven-plugin]updating poms for unomi_70_es_5_x version
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/544d2d4b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/544d2d4b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/544d2d4b
Branch: refs/heads/master
Commit: 544d2d4baa5e92172f2f9baa96336dd465c987de
Parents: 26f2ef2
Author: Serge Huber <sh...@apache.org>
Authored: Mon Jan 9 17:11:28 2017 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Mon Jan 9 17:11:28 2017 +0100
----------------------------------------------------------------------
api/pom.xml | 2 +-
extensions/geonames/pom.xml | 2 +-
extensions/geonames/rest/pom.xml | 8 +++---
extensions/geonames/services/pom.xml | 8 +++---
extensions/lists-extension/pom.xml | 4 +--
extensions/lists-extension/rest/pom.xml | 8 +++---
extensions/lists-extension/services/pom.xml | 8 +++---
extensions/pom.xml | 2 +-
extensions/privacy-extension/pom.xml | 4 +--
extensions/privacy-extension/rest/pom.xml | 8 +++---
extensions/privacy-extension/services/pom.xml | 8 +++---
itests/pom.xml | 8 +++---
kar/pom.xml | 30 ++++++++++----------
package/pom.xml | 4 +--
performance-tests/pom.xml | 4 +--
persistence-elasticsearch/core/pom.xml | 6 ++--
persistence-elasticsearch/plugins/pom.xml | 2 +-
.../plugins/security/pom.xml | 2 +-
persistence-elasticsearch/pom.xml | 2 +-
persistence-spi/pom.xml | 4 +--
plugins/baseplugin/pom.xml | 6 ++--
plugins/hover-event/pom.xml | 4 +--
plugins/mail/pom.xml | 4 +--
plugins/optimization-test/pom.xml | 2 +-
plugins/past-event/pom.xml | 6 ++--
plugins/pom.xml | 4 +--
plugins/request/pom.xml | 2 +-
plugins/tracked-event/pom.xml | 6 ++--
pom.xml | 2 +-
rest/pom.xml | 6 ++--
samples/pom.xml | 2 +-
samples/tweet-button-plugin/pom.xml | 4 +--
services/pom.xml | 6 ++--
wab/pom.xml | 6 ++--
34 files changed, 92 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/api/pom.xml
----------------------------------------------------------------------
diff --git a/api/pom.xml b/api/pom.xml
index cdf4fd8..0f06620 100644
--- a/api/pom.xml
+++ b/api/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/extensions/geonames/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/geonames/pom.xml b/extensions/geonames/pom.xml
index 61ae8f9..f601a7c 100644
--- a/extensions/geonames/pom.xml
+++ b/extensions/geonames/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-extensions</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>cxs-geonames</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/extensions/geonames/rest/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/geonames/rest/pom.xml b/extensions/geonames/rest/pom.xml
index ff52075..cec5546 100644
--- a/extensions/geonames/rest/pom.xml
+++ b/extensions/geonames/rest/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-geonames</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -33,14 +33,14 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-geonames-services</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
@@ -76,7 +76,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/extensions/geonames/services/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/geonames/services/pom.xml b/extensions/geonames/services/pom.xml
index 3c1fc0b..e1e7310 100644
--- a/extensions/geonames/services/pom.xml
+++ b/extensions/geonames/services/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-geonames</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -28,20 +28,20 @@
<artifactId>cxs-geonames-services</artifactId>
<name>Apache Unomi :: Extensions :: Geonames Database :: Service</name>
<description>Service implementation for the Apache Unomi Context Server extension that integrates with the Geonames database</description>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<packaging>bundle</packaging>
<dependencies>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/extensions/lists-extension/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/lists-extension/pom.xml b/extensions/lists-extension/pom.xml
index b5bd423..d348ad9 100644
--- a/extensions/lists-extension/pom.xml
+++ b/extensions/lists-extension/pom.xml
@@ -27,12 +27,12 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-extensions</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>cxs-lists-extension</artifactId>
<name>Apache Unomi :: Extensions :: Lists</name>
<description>List extension for the Apache Unomi Context Server</description>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/extensions/lists-extension/rest/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/lists-extension/rest/pom.xml b/extensions/lists-extension/rest/pom.xml
index f2d31ad..e4e9414 100644
--- a/extensions/lists-extension/rest/pom.xml
+++ b/extensions/lists-extension/rest/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>cxs-lists-extension</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -34,14 +34,14 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-lists-extension-services</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
@@ -77,7 +77,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/extensions/lists-extension/services/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/lists-extension/services/pom.xml b/extensions/lists-extension/services/pom.xml
index 75f65f9..d7ebedb 100644
--- a/extensions/lists-extension/services/pom.xml
+++ b/extensions/lists-extension/services/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>cxs-lists-extension</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -28,20 +28,20 @@
<name>Apache Unomi :: Extensions :: Lists :: Service</name>
<description>List extension service implementation for the Apache Unomi Context Server</description>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<packaging>bundle</packaging>
<dependencies>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/extensions/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/pom.xml b/extensions/pom.xml
index e556570..aa9cb48 100644
--- a/extensions/pom.xml
+++ b/extensions/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-extensions</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/extensions/privacy-extension/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/privacy-extension/pom.xml b/extensions/privacy-extension/pom.xml
index b23038c..46e4315 100644
--- a/extensions/privacy-extension/pom.xml
+++ b/extensions/privacy-extension/pom.xml
@@ -27,12 +27,12 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-extensions</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>cxs-privacy-extension</artifactId>
<name>Apache Unomi :: Extensions :: Privacy</name>
<description>Privacy management extension for the Apache Unomi Context Server</description>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/extensions/privacy-extension/rest/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/privacy-extension/rest/pom.xml b/extensions/privacy-extension/rest/pom.xml
index 7c51644..0388a60 100644
--- a/extensions/privacy-extension/rest/pom.xml
+++ b/extensions/privacy-extension/rest/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>cxs-privacy-extension</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -33,14 +33,14 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-privacy-extension-services</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
@@ -76,7 +76,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/extensions/privacy-extension/services/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/privacy-extension/services/pom.xml b/extensions/privacy-extension/services/pom.xml
index b685e84..4afcde8 100644
--- a/extensions/privacy-extension/services/pom.xml
+++ b/extensions/privacy-extension/services/pom.xml
@@ -20,27 +20,27 @@
<parent>
<artifactId>cxs-privacy-extension</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>cxs-privacy-extension-services</artifactId>
<name>Apache Unomi :: Extensions :: Privacy :: Services</name>
<description>Privacy management extension service implementation for the Apache Unomi Context Server</description>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<packaging>bundle</packaging>
<dependencies>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/itests/pom.xml
----------------------------------------------------------------------
diff --git a/itests/pom.xml b/itests/pom.xml
index 0bde83c..faf303b 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-itests</artifactId>
<name>Apache Unomi :: Integration Tests</name>
@@ -31,19 +31,19 @@
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-kar</artifactId>
<classifier>features</classifier>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<type>xml</type>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-wab</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/kar/pom.xml
----------------------------------------------------------------------
diff --git a/kar/pom.xml b/kar/pom.xml
index 8a1c1c4..e70a10c 100644
--- a/kar/pom.xml
+++ b/kar/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-kar</artifactId>
@@ -33,72 +33,72 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-wab</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-services</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins-base</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins-request</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins-mail</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins-optimization-test</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-lists-extension-services</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-lists-extension-rest</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-geonames-services</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-geonames-rest</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-privacy-extension-services</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-privacy-extension-rest</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.servicemix.bundles</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/package/pom.xml
----------------------------------------------------------------------
diff --git a/package/pom.xml b/package/pom.xml
index fd630fd..b2e2c91 100644
--- a/package/pom.xml
+++ b/package/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi</artifactId>
@@ -84,7 +84,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-kar</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<classifier>features</classifier>
<type>xml</type>
<scope>runtime</scope>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/performance-tests/pom.xml
----------------------------------------------------------------------
diff --git a/performance-tests/pom.xml b/performance-tests/pom.xml
index 9a6b4ad..72c7abd 100644
--- a/performance-tests/pom.xml
+++ b/performance-tests/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-performance-tests</artifactId>
<name>Apache Unomi :: Performance Tests</name>
@@ -30,7 +30,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-wab</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/persistence-elasticsearch/core/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/pom.xml b/persistence-elasticsearch/core/pom.xml
index ad96cfb..dc9d5ed 100644
--- a/persistence-elasticsearch/core/pom.xml
+++ b/persistence-elasticsearch/core/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
@@ -44,13 +44,13 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/persistence-elasticsearch/plugins/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/plugins/pom.xml b/persistence-elasticsearch/plugins/pom.xml
index ccb075e..9e361c0 100644
--- a/persistence-elasticsearch/plugins/pom.xml
+++ b/persistence-elasticsearch/plugins/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-persistence-elasticsearch-plugins</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/persistence-elasticsearch/plugins/security/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/plugins/security/pom.xml b/persistence-elasticsearch/plugins/security/pom.xml
index 1028e95..b029a8c 100644
--- a/persistence-elasticsearch/plugins/security/pom.xml
+++ b/persistence-elasticsearch/plugins/security/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-plugins</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-persistence-elasticsearch-plugins-security</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/persistence-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/pom.xml b/persistence-elasticsearch/pom.xml
index 3cace61..99ef177 100644
--- a/persistence-elasticsearch/pom.xml
+++ b/persistence-elasticsearch/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-persistence-elasticsearch</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/persistence-spi/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-spi/pom.xml b/persistence-spi/pom.xml
index 50bd36f..68d5ba7 100644
--- a/persistence-spi/pom.xml
+++ b/persistence-spi/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-persistence-spi</artifactId>
@@ -33,7 +33,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/plugins/baseplugin/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/pom.xml b/plugins/baseplugin/pom.xml
index 4aa9d54..078cfb8 100644
--- a/plugins/baseplugin/pom.xml
+++ b/plugins/baseplugin/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-base</artifactId>
@@ -55,13 +55,13 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/plugins/hover-event/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/hover-event/pom.xml b/plugins/hover-event/pom.xml
index 980cb84..979aa35 100644
--- a/plugins/hover-event/pom.xml
+++ b/plugins/hover-event/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-hover-event</artifactId>
@@ -33,7 +33,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/plugins/mail/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/mail/pom.xml b/plugins/mail/pom.xml
index 146c9c4..2bdcdd3 100644
--- a/plugins/mail/pom.xml
+++ b/plugins/mail/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-mail</artifactId>
@@ -33,7 +33,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/plugins/optimization-test/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/optimization-test/pom.xml b/plugins/optimization-test/pom.xml
index 04926da..f034ed2 100644
--- a/plugins/optimization-test/pom.xml
+++ b/plugins/optimization-test/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-optimization-test</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/plugins/past-event/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/past-event/pom.xml b/plugins/past-event/pom.xml
index 520808f..7f63535 100644
--- a/plugins/past-event/pom.xml
+++ b/plugins/past-event/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-past-event</artifactId>
@@ -33,13 +33,13 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/plugins/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/pom.xml b/plugins/pom.xml
index 77d9535..35d1fbd 100644
--- a/plugins/pom.xml
+++ b/plugins/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins</artifactId>
@@ -43,7 +43,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/plugins/request/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/request/pom.xml b/plugins/request/pom.xml
index 08e7a5a..1764ed1 100644
--- a/plugins/request/pom.xml
+++ b/plugins/request/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-request</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/plugins/tracked-event/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/tracked-event/pom.xml b/plugins/tracked-event/pom.xml
index 874e723..bb47811 100644
--- a/plugins/tracked-event/pom.xml
+++ b/plugins/tracked-event/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-tracked-event</artifactId>
@@ -33,13 +33,13 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6de2441..1f6cc55 100644
--- a/pom.xml
+++ b/pom.xml
@@ -34,7 +34,7 @@
provides a high-performance user profile and event tracking server.
</description>
<url>http://unomi.incubator.apache.org</url>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<licenses>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/rest/pom.xml
----------------------------------------------------------------------
diff --git a/rest/pom.xml b/rest/pom.xml
index 4d471a6..8ba9267 100644
--- a/rest/pom.xml
+++ b/rest/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-rest</artifactId>
@@ -46,7 +46,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
@@ -88,7 +88,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<!--<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/samples/pom.xml
----------------------------------------------------------------------
diff --git a/samples/pom.xml b/samples/pom.xml
index 20a0403..8941427 100644
--- a/samples/pom.xml
+++ b/samples/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>unomi-root</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/samples/tweet-button-plugin/pom.xml
----------------------------------------------------------------------
diff --git a/samples/tweet-button-plugin/pom.xml b/samples/tweet-button-plugin/pom.xml
index 8d5076c..c1725d5 100644
--- a/samples/tweet-button-plugin/pom.xml
+++ b/samples/tweet-button-plugin/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>samples</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>tweet-button-plugin</artifactId>
@@ -33,7 +33,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/services/pom.xml
----------------------------------------------------------------------
diff --git a/services/pom.xml b/services/pom.xml
index 5188aea..124c220 100644
--- a/services/pom.xml
+++ b/services/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-services</artifactId>
@@ -33,7 +33,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
@@ -52,7 +52,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/544d2d4b/wab/pom.xml
----------------------------------------------------------------------
diff --git a/wab/pom.xml b/wab/pom.xml
index d89cceb..5af699f 100644
--- a/wab/pom.xml
+++ b/wab/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>unomi-wab</artifactId>
@@ -42,13 +42,13 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
+ <version>1.2.0-incubating-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
[02/50] [abbrv] incubator-unomi git commit: Merge remote-tracking
branch 'origin/master' into UNOMI-28-ES-2-X-UPGRADE
Posted by sh...@apache.org.
Merge remote-tracking branch 'origin/master' into UNOMI-28-ES-2-X-UPGRADE
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/ca1462d8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/ca1462d8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/ca1462d8
Branch: refs/heads/master
Commit: ca1462d866b27cc41f3bb46c81865b2174748d26
Parents: a7fb48d b46a599
Author: Serge Huber <sh...@apache.org>
Authored: Wed Nov 16 11:25:11 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Wed Nov 16 11:25:11 2016 +0100
----------------------------------------------------------------------
BUILDING | 23 +++++++++++++++++++-
.../actions/SetRemoteHostInfoAction.java | 3 ++-
pom.xml | 20 +++++++++++++++++
.../geographic/sessionAdminSubDiv1.json | 8 +++++++
.../geographic/sessionAdminSubDiv2.json | 8 +++++++
5 files changed, 60 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ca1462d8/pom.xml
----------------------------------------------------------------------
[15/50] [abbrv] incubator-unomi git commit: UNOMI 70 - Upgrade to
ElasticSearch 5.0 - Upgrade to ElasticSearch 5.0.1 - Upgrade CXF to 3.0.2 -
Upgrade to Karaf 3.0.8 - Replace Unomi clustering from ES clustering to Karaf
Cellar clustering
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
index 6089499..939120d 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
@@ -4,9 +4,9 @@
{
"all": {
"match": "*",
- "match_mapping_type": "string",
+ "match_mapping_type": "text",
"mapping": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
}
}
@@ -14,34 +14,325 @@
],
"properties": {
"eventType": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"itemId": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"itemType": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"profileId": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
+ },
+ "properties": {
+ "properties": {
+ "src_terms[0]": {
+ "properties": {
+ "term": {
+ "type": "keyword"
+ }
+ }
+ }
+ }
},
"scope": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"sessionId": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"timeStamp": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
+ },
+ "source": {
+ "properties": {
+ "eventType": {
+ "type": "keyword"
+ },
+ "itemId": {
+ "type": "keyword"
+ },
+ "itemType": {
+ "type": "keyword"
+ },
+ "profileId": {
+ "type": "keyword"
+ },
+ "properties": {
+ "properties": {
+ "pageInfo": {
+ "properties": {
+ "destinationURL": {
+ "type": "keyword"
+ },
+ "language": {
+ "type": "keyword"
+ },
+ "pageName": {
+ "type": "keyword"
+ },
+ "pagePath": {
+ "type": "keyword"
+ },
+ "referringURL": {
+ "type": "keyword"
+ },
+ "variant": {
+ "type": "keyword"
+ }
+ }
+ }
+ }
+ },
+ "scope": {
+ "type": "keyword"
+ },
+ "sessionId": {
+ "type": "keyword"
+ },
+ "source": {
+ "properties": {
+ "itemId": {
+ "type": "keyword"
+ },
+ "itemType": {
+ "type": "keyword"
+ },
+ "scope": {
+ "type": "keyword"
+ }
+ }
+ },
+ "target": {
+ "properties": {
+ "itemId": {
+ "type": "keyword"
+ },
+ "itemType": {
+ "type": "keyword"
+ },
+ "properties": {
+ "properties": {
+ "pageInfo": {
+ "properties": {
+ "destinationURL": {
+ "type": "keyword"
+ },
+ "language": {
+ "type": "keyword"
+ },
+ "pageName": {
+ "type": "keyword"
+ },
+ "pagePath": {
+ "type": "keyword"
+ },
+ "referringURL": {
+ "type": "keyword"
+ },
+ "variant": {
+ "type": "keyword"
+ }
+ }
+ }
+ }
+ },
+ "scope": {
+ "type": "keyword"
+ }
+ }
+ },
+ "timeStamp": {
+ "type": "date"
+ }
+ }
+ },
+ "target": {
+ "properties": {
+ "campaignId": {
+ "type": "keyword"
+ },
+ "duration": {
+ "type": "long"
+ },
+ "itemId": {
+ "type": "keyword"
+ },
+ "itemType": {
+ "type": "keyword"
+ },
+ "metadata": {
+ "properties": {
+ "enabled": {
+ "type": "boolean"
+ },
+ "hidden": {
+ "type": "boolean"
+ },
+ "id": {
+ "type": "keyword"
+ },
+ "missingPlugins": {
+ "type": "boolean"
+ },
+ "name": {
+ "type": "keyword"
+ },
+ "readOnly": {
+ "type": "boolean"
+ },
+ "scope": {
+ "type": "keyword"
+ },
+ "tags": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ },
+ "profile": {
+ "properties": {
+ "itemId": {
+ "type": "keyword"
+ },
+ "itemType": {
+ "type": "keyword"
+ },
+ "properties": {
+ "properties": {
+ "firstVisit": {
+ "type": "date"
+ }
+ }
+ }
+ }
+ },
+ "profileId": {
+ "type": "keyword"
+ },
+ "properties": {
+ "properties": {
+ "birthDate": {
+ "type": "date"
+ },
+ "email": {
+ "type": "keyword"
+ },
+ "faceBookId": {
+ "type": "keyword"
+ },
+ "firstName": {
+ "type": "keyword"
+ },
+ "gender": {
+ "type": "keyword"
+ },
+ "goalId": {
+ "type": "keyword"
+ },
+ "lastName": {
+ "type": "keyword"
+ },
+ "leadAssignedTo": {
+ "type": "keyword"
+ },
+ "nationality": {
+ "type": "keyword"
+ },
+ "origin": {
+ "type": "keyword"
+ },
+ "pageInfo": {
+ "properties": {
+ "destinationURL": {
+ "type": "keyword"
+ },
+ "language": {
+ "type": "keyword"
+ },
+ "pageName": {
+ "type": "keyword"
+ },
+ "pagePath": {
+ "type": "keyword"
+ },
+ "referringURL": {
+ "type": "keyword"
+ },
+ "variant": {
+ "type": "keyword"
+ }
+ }
+ },
+ "path": {
+ "type": "keyword"
+ },
+ "phoneNumber": {
+ "type": "keyword"
+ },
+ "randomNumber": {
+ "type": "long"
+ },
+ "twitterId": {
+ "type": "keyword"
+ },
+ "variantId": {
+ "type": "keyword"
+ },
+ "workspace": {
+ "type": "keyword"
+ }
+ }
+ },
+ "scope": {
+ "type": "keyword"
+ },
+ "size": {
+ "type": "long"
+ },
+ "startEvent": {
+ "properties": {
+ "parameterValues": {
+ "properties": {
+ "pagePath": {
+ "type": "keyword"
+ }
+ }
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ },
+ "targetEvent": {
+ "properties": {
+ "parameterValues": {
+ "properties": {
+ "pagePath": {
+ "type": "keyword"
+ }
+ }
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ },
+ "timeStamp": {
+ "type": "date"
+ }
+ }
}
}
}
}
+
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
index 0fb7bcf..7290220 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
@@ -4,9 +4,9 @@
{
"all": {
"match": "*",
- "match_mapping_type": "string",
+ "match_mapping_type": "text",
"mapping": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
}
}
@@ -14,22 +14,19 @@
],
"properties": {
"campaignId": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"itemId": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"itemType": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"metadata": {
"properties": {
"description": {
- "type": "string",
- "analyzer": "folding"
+ "type": "text",
+ "analyzer" : "folding"
},
"enabled": {
"type": "boolean"
@@ -38,25 +35,22 @@
"type": "boolean"
},
"id": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"missingPlugins": {
"type": "boolean"
},
"name": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"readOnly": {
"type": "boolean"
},
"scope": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"tags": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/persona.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/persona.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/persona.json
index d04715f..db9828b 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/persona.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/persona.json
@@ -4,13 +4,29 @@
{
"all": {
"match": "*",
- "match_mapping_type": "string",
+ "match_mapping_type": "text",
"mapping": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
}
}
}
- ]
+ ],
+ "properties" : {
+ "properties" : {
+ "properties": {
+ "description": {
+ "type": "text",
+ "analyzer": "folding"
+ },
+ "firstName": {
+ "type": "keyword"
+ },
+ "lastName": {
+ "type": "keyword"
+ }
+ }
+ }
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
index 6ba12aa..c54a860 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
@@ -4,27 +4,96 @@
{
"all": {
"match": "*",
- "match_mapping_type": "string",
+ "match_mapping_type": "text",
"mapping": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
}
}
}
],
"properties": {
+ "duration" : {
+ "type" : "long"
+ },
+ "itemId" : {
+ "type" : "keyword"
+ },
+ "itemType" : {
+ "type" : "keyword"
+ },
"timeStamp": {
"type": "date"
},
"lastEventDate": {
"type": "date"
},
- "properties": {
- "properties": {
- "location": {
- "type": "geo_point"
+ "profile" : {
+ "properties" : {
+ "itemId" : {
+ "type" : "keyword"
+ },
+ "itemType" : {
+ "type" : "keyword"
+ },
+ "properties" : {
+ "properties" : {
+ "description" : {
+ "type" : "text",
+ "analyzer" : "folding"
+ },
+ "firstName" : {
+ "type" : "keyword"
+ },
+ "lastName" : {
+ "type" : "keyword"
+ }
+ }
+ }
+ }
+ },
+ "profileId" : {
+ "type" : "keyword"
+ },
+ "properties" : {
+ "properties" : {
+ "deviceCategory" : {
+ "type" : "keyword"
+ },
+ "location" : {
+ "type" : "geo_point"
+ },
+ "operatingSystemFamily" : {
+ "type" : "keyword"
+ },
+ "operatingSystemName" : {
+ "type" : "keyword"
+ },
+ "remoteAddr" : {
+ "type" : "keyword"
+ },
+ "remoteHost" : {
+ "type" : "keyword"
+ },
+ "sessionCity" : {
+ "type" : "keyword"
+ },
+ "sessionCountryCode" : {
+ "type" : "keyword"
+ },
+ "sessionCountryName" : {
+ "type" : "keyword"
+ },
+ "userAgentName" : {
+ "type" : "keyword"
+ },
+ "userAgentVersion" : {
+ "type" : "keyword"
}
}
+ },
+ "size" : {
+ "type" : "long"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
index 4a20426..9aba5d0 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
@@ -1,37 +1,53 @@
{
"profile": {
- "_all": {
- "enabled": true
- },
"dynamic_templates": [
{
"all": {
"match": "*",
- "match_mapping_type": "string",
+ "match_mapping_type": "text",
"mapping": {
- "type": "string",
- "analyzer": "folding",
- "include_in_all": true
+ "type": "text",
+ "analyzer": "folding"
}
}
}
],
"properties": {
"itemId": {
- "type": "string",
- "analyzer": "folding",
- "include_in_all": true
+ "type": "keyword"
},
"itemType": {
- "type": "string",
- "analyzer": "folding",
- "include_in_all": true
+ "type": "keyword"
},
"properties": {
"properties": {
+ "j:nodename": {
+ "type": "keyword"
+ },
+ "preferredLanguage": {
+ "type": "keyword"
+ },
"firstVisit": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
+ },
+ "lastVisit": {
+ "type": "date",
+ "format": "strict_date_optional_time||epoch_millis"
+ },
+ "previousVisit": {
+ "type": "date",
+ "format": "strict_date_optional_time||epoch_millis"
+ },
+ "nbOfVisits": {
+ "type": "long"
+ }
+ }
+ },
+ "systemProperties" : {
+ "properties" : {
+ "mergeIdentifier" : {
+ "type" : "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
index 88a958e..c03111c 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
@@ -4,9 +4,9 @@
{
"all": {
"match": "*",
- "match_mapping_type": "string",
+ "match_mapping_type": "text",
"mapping": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
}
}
@@ -14,40 +14,32 @@
],
"properties": {
"automaticMappingsFrom": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"dateRanges": {
"properties": {
"from": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"key": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"to": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
}
}
},
"defaultValue": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"itemId": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"itemType": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"mergeStrategy": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"metadata": {
"properties": {
@@ -58,15 +50,13 @@
"type": "boolean"
},
"id": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"missingPlugins": {
"type": "boolean"
},
"name": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"readOnly": {
"type": "boolean"
@@ -82,8 +72,7 @@
"type": "double"
},
"key": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"to": {
"type": "double"
@@ -97,16 +86,14 @@
"type": "double"
},
"tags": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
},
"target": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"type": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
index 67263cf..842181a 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
@@ -4,9 +4,9 @@
{
"all": {
"match": "*",
- "match_mapping_type": "string",
+ "match_mapping_type": "text",
"mapping": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
}
}
@@ -14,17 +14,15 @@
],
"properties": {
"itemId": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"itemType": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"metadata": {
"properties": {
"description": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
},
"enabled": {
@@ -34,22 +32,19 @@
"type": "boolean"
},
"id": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"missingPlugins": {
"type": "boolean"
},
"name": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"readOnly": {
"type": "boolean"
},
"scope": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
}
}
},
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
index b62d0a6..70b2f8d 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
@@ -4,9 +4,9 @@
{
"all": {
"match": "*",
- "match_mapping_type": "string",
+ "match_mapping_type": "text",
"mapping": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
}
}
@@ -14,17 +14,15 @@
],
"properties": {
"itemId": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"itemType": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"metadata": {
"properties": {
"description": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
},
"enabled": {
@@ -34,26 +32,23 @@
"type": "boolean"
},
"id": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"missingPlugins": {
"type": "boolean"
},
"name": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"readOnly": {
"type": "boolean"
},
"scope": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"tags": {
- "type": "string",
- "analyzer": "folding"
+ "type": "text",
+ "analyzer" : "folding"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
index c4527a5..2d25939 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
@@ -4,9 +4,9 @@
{
"all": {
"match": "*",
- "match_mapping_type": "string",
+ "match_mapping_type": "text",
"mapping": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
}
}
@@ -14,17 +14,15 @@
],
"properties": {
"itemId": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"itemType": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"metadata": {
"properties": {
"description": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
},
"enabled": {
@@ -34,22 +32,19 @@
"type": "boolean"
},
"id": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"missingPlugins": {
"type": "boolean"
},
"name": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"readOnly": {
"type": "boolean"
},
"scope": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
index 2047b4e..b0f6215 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
@@ -4,9 +4,9 @@
{
"all": {
"match": "*",
- "match_mapping_type": "string",
+ "match_mapping_type": "text",
"mapping": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
}
}
@@ -17,38 +17,47 @@
"type": "long"
},
"itemId": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"itemType": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"lastEventDate": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"profileId": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
+ },
+ "profile": {
+ "properties": {
+ "itemId": {
+ "type": "keyword"
+ },
+ "itemType": {
+ "type": "keyword"
+ }
+ }
},
"properties": {
"properties": {
"location": {
"type": "geo_point"
+ },
+ "pageReferringURL": {
+ "type": "keyword"
+ },
+ "referringURL": {
+ "type": "keyword"
}
}
},
"scope": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"size": {
"type": "long"
},
- "systemProperties": {
- "type": "object"
- },
"timeStamp": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index 75039d8..23963ac 100644
--- a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -35,10 +35,6 @@
<cm:property name="elasticSearchConfig" value="file:${karaf.etc}/elasticsearch.yml"/>
<cm:property name="monthlyIndex.numberOfShards" value="3"/>
<cm:property name="monthlyIndex.numberOfReplicas" value="0"/>
- <cm:property name="numberOfShards" value="5"/>
- <cm:property name="numberOfReplicas" value="0"/>
- <cm:property name="discovery.zen.ping.multicast.enabled" value="false"/>
- <cm:property name="node.data" value="true"/>
<cm:property name="defaultQueryLimit" value="10"/>
<cm:property name="bulkProcessor.name" value="unomi-bulk" />
@@ -47,6 +43,11 @@
<cm:property name="bulkProcessor.bulkSize" value="5MB" />
<cm:property name="bulkProcessor.flushInterval" value="5s" />
<cm:property name="bulkProcessor.backoffPolicy" value="exponential" />
+
+ <cm:property name="cluster.group" value="default" />
+ <cm:property name="cluster.jmxUsername" value="karaf" />
+ <cm:property name="cluster.jmxPassword" value="karaf" />
+ <cm:property name="cluster.jmxPort" value="1099" />
</cm:default-properties>
</cm:property-placeholder>
@@ -60,6 +61,11 @@
</cm:default-properties>
</cm:property-placeholder>
+ <reference id="karafCellarClusterManager" interface="org.apache.karaf.cellar.core.ClusterManager" />
+ <reference id="karafCellarEventProducer" interface="org.apache.karaf.cellar.core.event.EventProducer" />
+ <reference id="karafCellarGroupManager" interface="org.apache.karaf.cellar.core.GroupManager" />
+ <reference id="osgiConfigurationAdmin" interface="org.osgi.service.cm.ConfigurationAdmin"/>
+
<service id="elasticSearchPersistenceService" ref="elasticSearchPersistenceServiceImpl">
<interfaces>
<value>org.apache.unomi.persistence.spi.PersistenceService</value>
@@ -88,10 +94,6 @@
<property name="indexName" value="${es.index.name}"/>
<property name="monthlyIndexNumberOfShards" value="${es.monthlyIndex.numberOfShards}"/>
<property name="monthlyIndexNumberOfReplicas" value="${es.monthlyIndex.numberOfReplicas}"/>
- <property name="numberOfShards" value="${es.numberOfShards}"/>
- <property name="numberOfReplicas" value="${es.numberOfReplicas}"/>
- <property name="nodeData" value="${es.node.data}"/>
- <property name="discoveryEnabled" value="${es.discovery.zen.ping.multicast.enabled}"/>
<property name="address" value="${web.contextserver.address}"/>
<property name="port" value="${web.contextserver.port}"/>
<property name="secureAddress" value="${web.contextserver.secureAddress}"/>
@@ -119,6 +121,14 @@
<property name="bulkProcessorBulkSize" value="${es.bulkProcessor.bulkSize}" />
<property name="bulkProcessorFlushInterval" value="${es.bulkProcessor.flushInterval}" />
<property name="bulkProcessorBackoffPolicy" value="${es.bulkProcessor.backoffPolicy}" />
+ <property name="karafCellarClusterManager" ref="karafCellarClusterManager" />
+ <property name="karafCellarEventProducer" ref="karafCellarEventProducer" />
+ <property name="karafCellarGroupManager" ref="karafCellarGroupManager" />
+ <property name="karafCellarGroupName" value="${es.cluster.group}" />
+ <property name="osgiConfigurationAdmin" ref="osgiConfigurationAdmin" />
+ <property name="karafJMXUsername" value="${es.cluster.jmxUsername}" />
+ <property name="karafJMXPassword" value="${es.cluster.jmxPassword}" />
+ <property name="karafJMXPort" value="${es.cluster.jmxPort}" />
</bean>
</blueprint>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/log4j2.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/log4j2.xml b/persistence-elasticsearch/core/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..ccafeef
--- /dev/null
+++ b/persistence-elasticsearch/core/src/main/resources/log4j2.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<Configuration status="WARN">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
index 2b193f6..ddcbed5 100644
--- a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
+++ b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
@@ -35,4 +35,9 @@ bulkProcessor.concurrentRequests=1
bulkProcessor.bulkActions=1000
bulkProcessor.bulkSize=5MB
bulkProcessor.flushInterval=5s
-bulkProcessor.backoffPolicy=exponential
\ No newline at end of file
+bulkProcessor.backoffPolicy=exponential
+
+cluster.group=default
+cluster.jmxUsername=karaf
+cluster.jmxPassword=karaf
+cluster.jmxPort=1099
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPlugin.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPlugin.java b/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPlugin.java
index 79fc7d3..b293527 100644
--- a/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPlugin.java
+++ b/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPlugin.java
@@ -22,9 +22,9 @@ import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.plugins.Plugin;
+import java.util.ArrayList;
import java.util.Collection;
-
-import static com.google.common.collect.Lists.newArrayList;
+import java.util.Collections;
public class SecurityPlugin extends Plugin {
@@ -32,30 +32,22 @@ public class SecurityPlugin extends Plugin {
super();
}
- public String name() {
- return "contextserver-security";
- }
-
- public String description() {
- return "A plugin that provides some basic security to the Context Server elasticsearch HTTP and Transport connectors";
- }
-
@Override
- public Collection<Module> nodeModules() {
- Collection<Module> modules = newArrayList();
- // if (settings.getAsBoolean("security.enabled", true)) {
- modules.add(new SecurityPluginModule());
- // }
- return modules;
+ public Collection<Module> createGuiceModules() {
+ return Collections.<Module>singletonList(new SecurityPluginModule());
}
- @SuppressWarnings("rawtypes")
@Override
- public Collection<Class<? extends LifecycleComponent>> nodeServices() {
- Collection<Class<? extends LifecycleComponent>> services = newArrayList();
- // if (settings.getAsBoolean("security.enabled", true)) {
+ @SuppressWarnings("rawtypes") // Plugin use a rawtype
+ public Collection<Class<? extends LifecycleComponent>> getGuiceServiceClasses() {
+ Collection<Class<? extends LifecycleComponent>> services = new ArrayList<>();
services.add(SecurityPluginService.class);
- // }
return services;
}
+
+ @Override
+ public Settings additionalSettings() {
+ return Settings.EMPTY;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPluginService.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPluginService.java b/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPluginService.java
index 6bd73eb..7c99439 100644
--- a/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPluginService.java
+++ b/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPluginService.java
@@ -18,6 +18,7 @@
package org.apache.unomi.elasticsearch.plugin.security;
import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.Inject;
@@ -37,7 +38,7 @@ import java.util.List;
/**
* ElasticSearch plugin that simply rejects connection from non-authorized IP ranges
*/
-public class SecurityPluginService extends AbstractLifecycleComponent<SecurityPluginService> {
+public class SecurityPluginService extends AbstractLifecycleComponent {
private static final Logger logger = LoggerFactory.getLogger(SecurityPluginService.class.getName());
@@ -102,7 +103,7 @@ public class SecurityPluginService extends AbstractLifecycleComponent<SecurityPl
protected void doStart() throws ElasticsearchException {
restFilter = new RestFilter() {
@Override
- public void process(RestRequest request, RestChannel channel, RestFilterChain filterChain) throws Exception {
+ public void process(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception {
logger.info("Processing REST request=" + request + " channel=" + channel);
if (request.getRemoteAddress() instanceof InetSocketAddress) {
InetSocketAddress inetSocketAddress = (InetSocketAddress) request.getRemoteAddress();
@@ -113,7 +114,7 @@ public class SecurityPluginService extends AbstractLifecycleComponent<SecurityPl
} else {
logger.warn("Unexpected SocketAddress that is not an InetSocketAddress (but an instance of " + request.getRemoteAddress().getClass().getName() + "), IP range filtering is DISABLED !");
}
- filterChain.continueProcessing(request, channel);
+ filterChain.continueProcessing(request, channel, client);
}
};
restController.registerFilter(restFilter);
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
----------------------------------------------------------------------
diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
index 90b0efc..d81be03 100644
--- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
+++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
@@ -178,14 +178,6 @@ public interface PersistenceService {
void createMapping(String type, String source);
/**
- * TODO
- *
- * @param item
- * @return
- */
- List<String> getMatchingSavedQueries(Item item);
-
- /**
* Checks whether the specified item satisfies the provided condition.
*
* TODO: rename to isMatching?
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java
index f5330ed..94e8519 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java
@@ -23,6 +23,8 @@ import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBui
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Map;
@@ -32,6 +34,8 @@ import java.util.Map;
*/
public class BooleanConditionESQueryBuilder implements ConditionESQueryBuilder {
+ private static final Logger logger = LoggerFactory.getLogger(BooleanConditionESQueryBuilder.class.getName());
+
@Override
public QueryBuilder buildQuery(Condition condition, Map<String, Object> context,
ConditionESQueryBuilderDispatcher dispatcher) {
@@ -48,9 +52,19 @@ public class BooleanConditionESQueryBuilder implements ConditionESQueryBuilder {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
for (int i = 0; i < conditionCount; i++) {
if (isAndOperator) {
- boolQueryBuilder.must(dispatcher.buildFilter(conditions.get(i)));
+ QueryBuilder andFilter = dispatcher.buildFilter(conditions.get(i));
+ if (andFilter != null) {
+ boolQueryBuilder.must(andFilter);
+ } else {
+ logger.warn("Null filter for boolean AND sub condition " + conditions.get(i));
+ }
} else {
- boolQueryBuilder.should(dispatcher.buildFilter(conditions.get(i)));
+ QueryBuilder orFilter = dispatcher.buildFilter(conditions.get(i));
+ if (orFilter != null) {
+ boolQueryBuilder.should(orFilter);
+ } else {
+ logger.warn("Null filter for boolean OR sub condition " + conditions.get(i));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java
index 083cfe4..001bd3d 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java
@@ -37,8 +37,7 @@ public class GeoLocationByPointSessionConditionESQueryBuilder implements Conditi
if(circleLatitude != null && circleLongitude != null && distance != null) {
return QueryBuilders.geoDistanceQuery("location")
- .lat(circleLatitude)
- .lon(circleLongitude)
+ .point(circleLatitude, circleLongitude)
.distance(distance);
}
} else if("rectangle".equals(type)) {
@@ -49,8 +48,7 @@ public class GeoLocationByPointSessionConditionESQueryBuilder implements Conditi
if(rectLatitudeNE != null && rectLongitudeNE != null && rectLatitudeSW != null && rectLongitudeSW != null) {
return QueryBuilders.geoBoundingBoxQuery("location")
- .topLeft(rectLatitudeNE, rectLongitudeNE)
- .bottomRight(rectLatitudeSW, rectLongitudeSW);
+ .setCorners(rectLatitudeNE, rectLongitudeNE,rectLatitudeSW, rectLongitudeSW);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
index 4db6e1a..67404a8 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
@@ -34,7 +34,7 @@ import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorD
import org.apache.unomi.persistence.spi.PropertyHelper;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.joda.DateMathParser;
-import org.elasticsearch.index.mapper.core.DateFieldMapper;
+import org.elasticsearch.index.mapper.DateFieldMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -280,7 +280,7 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
if (value instanceof Date) {
return ((Date) value);
} else {
- DateMathParser parser = new DateMathParser(DateFieldMapper.Defaults.DATE_TIME_FORMATTER);
+ DateMathParser parser = new DateMathParser(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER);
try {
return new Date(parser.parse(value.toString(), new Callable<Long>() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index fd75b01..1d8b334 100644
--- a/pom.xml
+++ b/pom.xml
@@ -63,12 +63,13 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <cxf.version>2.7.11</cxf.version>
+ <cxf.version>3.0.2</cxf.version>
<version.jackson.core>2.6.6</version.jackson.core> <!-- version used by CXF -->
<version.jackson.jaxb>2.6.6</version.jackson.jaxb> <!-- version used by CXF -->
- <version.karaf>3.0.2</version.karaf>
+ <version.karaf>3.0.8</version.karaf>
+ <version.karaf.cellar>3.0.3</version.karaf.cellar>
<version.pax.exam>4.9.1</version.pax.exam>
- <elasticsearch.version>2.4.1</elasticsearch.version>
+ <elasticsearch.version>5.0.1</elasticsearch.version>
<maven.compiler.source>1.7</maven.compiler.source>
<maven.compiler.target>1.7</maven.compiler.target>
@@ -521,11 +522,6 @@
<!-- Apache CXF dependencies -->
<dependency>
<groupId>org.apache.cxf</groupId>
- <artifactId>cxf-rt-core</artifactId>
- <version>${cxf.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.cxf</groupId>
<artifactId>cxf-rt-frontend-jaxws</artifactId>
<version>${cxf.version}</version>
</dependency>
@@ -648,6 +644,25 @@
<type>xml</type>
<scope>runtime</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.karaf.cellar</groupId>
+ <artifactId>apache-karaf-cellar</artifactId>
+ <classifier>features</classifier>
+ <version>${version.karaf.cellar}</version>
+ <type>xml</type>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.karaf.cellar</groupId>
+ <artifactId>org.apache.karaf.cellar.core</artifactId>
+ <version>${version.karaf.cellar}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.karaf.cellar</groupId>
+ <artifactId>org.apache.karaf.cellar.config</artifactId>
+ <version>${version.karaf.cellar}</version>
+ </dependency>
+
<!-- End of Apache Karaf dependencies -->
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/rest/pom.xml
----------------------------------------------------------------------
diff --git a/rest/pom.xml b/rest/pom.xml
index f795ff1..4d471a6 100644
--- a/rest/pom.xml
+++ b/rest/pom.xml
@@ -64,11 +64,6 @@
<dependency>
<groupId>org.apache.cxf</groupId>
- <artifactId>cxf-rt-core</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.cxf</groupId>
<artifactId>cxf-rt-frontend-jaxws</artifactId>
<scope>provided</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/services/pom.xml
----------------------------------------------------------------------
diff --git a/services/pom.xml b/services/pom.xml
index c90ebba..7a58cd7 100644
--- a/services/pom.xml
+++ b/services/pom.xml
@@ -64,16 +64,6 @@
<dependency>
<groupId>org.apache.cxf</groupId>
- <artifactId>cxf-rt-core</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.cxf</groupId>
- <artifactId>cxf-rt-frontend-jaxws</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.cxf</groupId>
<artifactId>cxf-rt-frontend-jaxrs</artifactId>
<scope>provided</scope>
</dependency>
[21/50] [abbrv] incubator-unomi git commit: Further optimizations -
Add shell script to build and run without integration and performance tests.
- Segment updates should now use a lot more memory since we now use scrolling
queries
Posted by sh...@apache.org.
Further optimizations
- Add shell script to build and run without integration and performance tests.
- Segment updates should now use a lot more memory since we now use scrolling queries
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/28d7dbde
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/28d7dbde
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/28d7dbde
Branch: refs/heads/master
Commit: 28d7dbde4728ed41eed6cc43022039ad02ddce7e
Parents: d7a7969
Author: Serge Huber <sh...@apache.org>
Authored: Thu Dec 15 16:22:38 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Thu Dec 15 16:22:38 2016 +0100
----------------------------------------------------------------------
.../java/org/apache/unomi/api/PartialList.java | 25 +++++
buildAndRunNoTests.sh | 34 ++++++
.../ElasticSearchPersistenceServiceImpl.java | 94 +++++++++++++---
.../persistence/spi/PersistenceService.java | 33 ++++++
.../PropertyConditionESQueryBuilder.java | 3 +-
.../services/services/SegmentServiceImpl.java | 111 ++++++++++++-------
.../resources/OSGI-INF/blueprint/blueprint.xml | 2 +
7 files changed, 242 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/28d7dbde/api/src/main/java/org/apache/unomi/api/PartialList.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/unomi/api/PartialList.java b/api/src/main/java/org/apache/unomi/api/PartialList.java
index 59f7e4d..daa69c9 100644
--- a/api/src/main/java/org/apache/unomi/api/PartialList.java
+++ b/api/src/main/java/org/apache/unomi/api/PartialList.java
@@ -37,6 +37,8 @@ public class PartialList<T> implements Serializable {
private long offset;
private long pageSize;
private long totalSize;
+ private String scrollIdentifier = null;
+ private String scrollTimeValidity = null;
/**
* Instantiates a new PartialList.
@@ -141,4 +143,27 @@ public class PartialList<T> implements Serializable {
return list.get(index);
}
+ /**
+ * Retrieve the scroll identifier to make it possible to continue a scrolling list query
+ * @return a string containing the scroll identifier, to be sent back in an subsequent request
+ */
+ public String getScrollIdentifier() {
+ return scrollIdentifier;
+ }
+
+ public void setScrollIdentifier(String scrollIdentifier) {
+ this.scrollIdentifier = scrollIdentifier;
+ }
+
+ /**
+ * Retrieve the value of the scroll time validity to make it possible to continue a scrolling list query
+ * @return a string containing a time value for the scroll validity, to be sent back in a subsequent request
+ */
+ public String getScrollTimeValidity() {
+ return scrollTimeValidity;
+ }
+
+ public void setScrollTimeValidity(String scrollTimeValidity) {
+ this.scrollTimeValidity = scrollTimeValidity;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/28d7dbde/buildAndRunNoTests.sh
----------------------------------------------------------------------
diff --git a/buildAndRunNoTests.sh b/buildAndRunNoTests.sh
new file mode 100755
index 0000000..c8f7641
--- /dev/null
+++ b/buildAndRunNoTests.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+################################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+echo Building...
+DIRNAME=`dirname "$0"`
+PROGNAME=`basename "$0"`
+if [ -f "$DIRNAME/setenv.sh" ]; then
+ . "$DIRNAME/setenv.sh"
+fi
+mvn clean install -P \!integration-tests,\!performance-tests,rat
+pushd package/target
+echo Uncompressing Unomi package...
+tar zxvf unomi-$UNOMI_VERSION.tar.gz
+cd unomi-$UNOMI_VERSION/bin
+echo Starting Unomi...
+./karaf debug
+popd
+
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/28d7dbde/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 794b03b..d6a136e 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -695,7 +695,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
@Override
public <T extends Item> PartialList<T> getAllItems(final Class<T> clazz, int offset, int size, String sortBy) {
- return query(QueryBuilders.matchAllQuery(), sortBy, clazz, offset, size, null);
+ return query(QueryBuilders.matchAllQuery(), sortBy, clazz, offset, size, null, null);
}
@Override
@@ -711,7 +711,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
String itemType = (String) clazz.getField("ITEM_TYPE").get(null);
if (itemsMonthlyIndexed.contains(itemType) && dateHint == null) {
- PartialList<T> r = query(QueryBuilders.idsQuery(itemType).ids(itemId), null, clazz, 0, 1, null);
+ PartialList<T> r = query(QueryBuilders.idsQuery(itemType).ids(itemId), null, clazz, 0, 1, null, null);
if (r.size() > 0) {
return r.get(0);
}
@@ -1128,12 +1128,17 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
@Override
public <T extends Item> PartialList<T> query(final Condition query, String sortBy, final Class<T> clazz, final int offset, final int size) {
- return query(conditionESQueryBuilderDispatcher.getQueryBuilder(query), sortBy, clazz, offset, size, null);
+ return query(conditionESQueryBuilderDispatcher.getQueryBuilder(query), sortBy, clazz, offset, size, null, null);
+ }
+
+ @Override
+ public <T extends Item> PartialList<T> query(final Condition query, String sortBy, final Class<T> clazz, final int offset, final int size, final String scrollTimeValidity) {
+ return query(conditionESQueryBuilderDispatcher.getQueryBuilder(query), sortBy, clazz, offset, size, null, scrollTimeValidity);
}
@Override
public <T extends Item> PartialList<T> queryFullText(final String fulltext, final Condition query, String sortBy, final Class<T> clazz, final int offset, final int size) {
- return query(QueryBuilders.boolQuery().must(QueryBuilders.queryStringQuery(fulltext).defaultField("_all")).must(conditionESQueryBuilderDispatcher.getQueryBuilder(query)), sortBy, clazz, offset, size, null);
+ return query(QueryBuilders.boolQuery().must(QueryBuilders.queryStringQuery(fulltext).defaultField("_all")).must(conditionESQueryBuilderDispatcher.getQueryBuilder(query)), sortBy, clazz, offset, size, null, null);
}
@Override
@@ -1143,22 +1148,22 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
@Override
public <T extends Item> List<T> query(final String fieldName, final String[] fieldValues, String sortBy, final Class<T> clazz) {
- return query(QueryBuilders.termsQuery(fieldName, ConditionContextHelper.foldToASCII(fieldValues)), sortBy, clazz, 0, -1, getRouting(fieldName, fieldValues, clazz)).getList();
+ return query(QueryBuilders.termsQuery(fieldName, ConditionContextHelper.foldToASCII(fieldValues)), sortBy, clazz, 0, -1, getRouting(fieldName, fieldValues, clazz), null).getList();
}
@Override
public <T extends Item> PartialList<T> query(String fieldName, String fieldValue, String sortBy, Class<T> clazz, int offset, int size) {
- return query(QueryBuilders.termQuery(fieldName, ConditionContextHelper.foldToASCII(fieldValue)), sortBy, clazz, offset, size, getRouting(fieldName, new String[]{fieldValue}, clazz));
+ return query(QueryBuilders.termQuery(fieldName, ConditionContextHelper.foldToASCII(fieldValue)), sortBy, clazz, offset, size, getRouting(fieldName, new String[]{fieldValue}, clazz), null);
}
@Override
public <T extends Item> PartialList<T> queryFullText(String fieldName, String fieldValue, String fulltext, String sortBy, Class<T> clazz, int offset, int size) {
- return query(QueryBuilders.boolQuery().must(QueryBuilders.queryStringQuery(fulltext).defaultField("_all")).must(QueryBuilders.termQuery(fieldName, fieldValue)), sortBy, clazz, offset, size, getRouting(fieldName, new String[]{fieldValue}, clazz));
+ return query(QueryBuilders.boolQuery().must(QueryBuilders.queryStringQuery(fulltext).defaultField("_all")).must(QueryBuilders.termQuery(fieldName, fieldValue)), sortBy, clazz, offset, size, getRouting(fieldName, new String[]{fieldValue}, clazz), null);
}
@Override
public <T extends Item> PartialList<T> queryFullText(String fulltext, String sortBy, Class<T> clazz, int offset, int size) {
- return query(QueryBuilders.queryStringQuery(fulltext).defaultField("_all"), sortBy, clazz, offset, size, getRouting("_all", new String[]{fulltext}, clazz));
+ return query(QueryBuilders.queryStringQuery(fulltext).defaultField("_all"), sortBy, clazz, offset, size, getRouting("_all", new String[]{fulltext}, clazz), null);
}
@Override
@@ -1166,7 +1171,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
RangeQueryBuilder builder = QueryBuilders.rangeQuery(fieldName);
builder.from(from);
builder.to(to);
- return query(builder, sortBy, clazz, offset, size, null);
+ return query(builder, sortBy, clazz, offset, size, null, null);
}
@Override
@@ -1193,21 +1198,35 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}.executeInClassLoader();
}
- private <T extends Item> PartialList<T> query(final QueryBuilder query, final String sortBy, final Class<T> clazz, final int offset, final int size, final String[] routing) {
+ private <T extends Item> PartialList<T> query(final QueryBuilder query, final String sortBy, final Class<T> clazz, final int offset, final int size, final String[] routing, final String scrollTimeValidity) {
return new InClassLoaderExecute<PartialList<T>>() {
@Override
protected PartialList<T> execute(Object... args) {
List<T> results = new ArrayList<T>();
+ String scrollIdentifier = null;
long totalHits = 0;
try {
String itemType = getItemType(clazz);
+ TimeValue keepAlive = TimeValue.timeValueHours(1);
+ SearchRequestBuilder requestBuilder = null;
+ if (scrollTimeValidity != null) {
+ keepAlive = TimeValue.parseTimeValue(scrollTimeValidity, TimeValue.timeValueHours(1), "scrollTimeValidity");
+ requestBuilder = client.prepareSearch(getIndexNameForQuery(itemType))
+ .setTypes(itemType)
+ .setFetchSource(true)
+ .setScroll(keepAlive)
+ .setFrom(offset)
+ .setQuery(query)
+ .setSize(size);
+ } else {
+ requestBuilder = client.prepareSearch(getIndexNameForQuery(itemType))
+ .setTypes(itemType)
+ .setFetchSource(true)
+ .setQuery(query)
+ .setFrom(offset);
+ }
- SearchRequestBuilder requestBuilder = client.prepareSearch(getIndexNameForQuery(itemType))
- .setTypes(itemType)
- .setFetchSource(true)
- .setQuery(query)
- .setFrom(offset);
if (size == Integer.MIN_VALUE) {
requestBuilder.setSize(defaultQueryLimit);
} else if (size != -1) {
@@ -1244,6 +1263,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
.execute()
.actionGet();
SearchHits searchHits = response.getHits();
+ scrollIdentifier = response.getScrollId();
totalHits = searchHits.getTotalHits();
for (SearchHit searchHit : searchHits) {
String sourceAsString = searchHit.getSourceAsString();
@@ -1255,7 +1275,49 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
logger.error("Error loading itemType=" + clazz.getName() + " query=" + query + " sortBy=" + sortBy, t);
}
- return new PartialList<T>(results, offset, size, totalHits);
+ PartialList<T> result = new PartialList<T>(results, offset, size, totalHits);
+ if (scrollIdentifier != null && totalHits != 0) {
+ result.setScrollIdentifier(scrollIdentifier);
+ result.setScrollTimeValidity(scrollTimeValidity);
+ }
+ return result;
+ }
+ }.executeInClassLoader();
+ }
+
+ @Override
+ public <T extends Item> PartialList<T> continueScrollQuery(final Class<T> clazz, final String scrollIdentifier, final String scrollTimeValidity) {
+ return new InClassLoaderExecute<PartialList<T>>() {
+
+ @Override
+ protected PartialList<T> execute(Object... args) {
+ List<T> results = new ArrayList<T>();
+ long totalHits = 0;
+ try {
+ TimeValue keepAlive = TimeValue.parseTimeValue(scrollTimeValidity, TimeValue.timeValueMinutes(10), "scrollTimeValidity");
+ SearchResponse response = client.prepareSearchScroll(scrollIdentifier).setScroll(keepAlive).execute().actionGet();
+
+ if (response.getHits().getHits().length == 0) {
+ client.prepareClearScroll().addScrollId(response.getScrollId()).execute().actionGet();
+ } else {
+ for (SearchHit searchHit : response.getHits().getHits()) {
+ // add hit to results
+ String sourceAsString = searchHit.getSourceAsString();
+ final T value = CustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz);
+ value.setItemId(searchHit.getId());
+ results.add(value);
+ }
+ }
+ PartialList<T> result = new PartialList<T>(results, 0, response.getHits().getHits().length, response.getHits().getTotalHits());
+ if (scrollIdentifier != null) {
+ result.setScrollIdentifier(scrollIdentifier);
+ result.setScrollTimeValidity(scrollTimeValidity);
+ }
+ return result;
+ } catch (Exception t) {
+ logger.error("Error continuing scrolling query for itemType=" + clazz.getName() + " scrollIdentifier=" + scrollIdentifier + " scrollTimeValidity=" + scrollTimeValidity, t);
+ }
+ return null;
}
}.executeInClassLoader();
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/28d7dbde/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
----------------------------------------------------------------------
diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
index 90b0efc..881c395 100644
--- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
+++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
@@ -294,6 +294,39 @@ public interface PersistenceService {
<T extends Item> PartialList<T> query(Condition query, String sortBy, Class<T> clazz, int offset, int size);
/**
+ * Retrieves a list of items satisfying the specified {@link Condition}, ordered according to the specified {@code sortBy} String and and paged: only {@code size} of them
+ * are retrieved, starting with the {@code offset}-th one. If a scroll identifier and time validity are specified, they will be used to perform a scrolling query, meaning
+ * that only partial results will be returned, but the scrolling can be continued.
+ *
+ * @param <T> the type of the Item subclass we want to retrieve
+ * @param query the {@link Condition} the items must satisfy to be retrieved
+ * @param sortBy an optional ({@code null} if no sorting is required) String of comma ({@code ,}) separated property names on which ordering should be performed, ordering
+ * elements according to the property order in the
+ * String, considering each in turn and moving on to the next one in case of equality of all preceding ones. Each property name is optionally followed by
+ * a column ({@code :}) and an order specifier: {@code asc} or {@code desc}.
+ * @param clazz the {@link Item} subclass of the items we want to retrieve
+ * @param offset zero or a positive integer specifying the position of the first item in the total ordered collection of matching items
+ * @param size a positive integer specifying how many matching items should be retrieved or {@code -1} if all of them should be retrieved. In the case of a scroll query
+ * this will be used as the scrolling window size.
+ * @param scrollTimeValidity the time the scrolling query should stay valid. This must contain a time unit value such as the ones supported by ElasticSearch, such as
+ * the ones declared here : https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#time-units
+ * @return a {@link PartialList} of items matching the specified criteria, with an scroll identifier and the scroll validity used if a scroll query was requested.
+ */
+ <T extends Item> PartialList<T> query(Condition query, String sortBy, Class<T> clazz, int offset, int size, String scrollTimeValidity);
+
+ /**
+ * Continues the execution of a scroll query, to retrieve the next results. If there are no more results the scroll query is also cleared.
+ * @param clazz the {@link Item} subclass of the items we want to retrieve
+ * @param scrollIdentifier a scroll identifier obtained by the execution of a first query and returned in the {@link PartialList} object
+ * @param scrollTimeValidity a scroll time validity value for the scroll query to stay valid. This must contain a time unit value such as the ones supported by ElasticSearch, such as
+ * the ones declared here : https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#time-units
+ * @param <T> the type of the Item subclass we want to retrieve
+ * @return a {@link PartialList} of items matching the specified criteria, with an scroll identifier and the scroll validity used if a scroll query was requested. Note that if
+ * there are no more results the list will be empty but not null.
+ */
+ <T extends Item> PartialList<T> continueScrollQuery(Class<T> clazz, String scrollIdentifier, String scrollTimeValidity);
+
+ /**
* Retrieves the same items as {@code query(query, sortBy, clazz, 0, -1)} with the added constraints that the matching elements must also have at least a field matching the
* specified full text query.
*
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/28d7dbde/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
index 7013581..c36722b 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
@@ -129,8 +129,9 @@ public class PropertyConditionESQueryBuilder implements ConditionESQueryBuilder
case "isNotDay":
checkRequiredValue(value, name, op, false);
return QueryBuilders.boolQuery().mustNot(getIsSameDayRange(value, name));
+ default:
+ throw new IllegalArgumentException("Impossible to build ES filter, unrecognized op=" + op);
}
- return null;
}
private void checkRequiredValuesSize(List<?> values, String name, String operator, int expectedSize) {
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/28d7dbde/services/src/main/java/org/apache/unomi/services/services/SegmentServiceImpl.java
----------------------------------------------------------------------
diff --git a/services/src/main/java/org/apache/unomi/services/services/SegmentServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/SegmentServiceImpl.java
index 3a64be1..2275d60 100644
--- a/services/src/main/java/org/apache/unomi/services/services/SegmentServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/services/SegmentServiceImpl.java
@@ -21,6 +21,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.unomi.api.*;
import org.apache.unomi.api.actions.Action;
import org.apache.unomi.api.conditions.Condition;
+import org.apache.unomi.api.conditions.ConditionType;
import org.apache.unomi.api.query.Query;
import org.apache.unomi.api.rules.Rule;
import org.apache.unomi.api.segments.*;
@@ -62,6 +63,7 @@ public class SegmentServiceImpl implements SegmentService, SynchronousBundleList
private List<Segment> allSegments;
private List<Scoring> allScoring;
private Timer segmentTimer;
+ private int segmentUpdateBatchSize = 1000;
public SegmentServiceImpl() {
logger.info("Initializing segment service...");
@@ -119,6 +121,10 @@ public class SegmentServiceImpl implements SegmentService, SynchronousBundleList
this.rulesService = rulesService;
}
+ public void setSegmentUpdateBatchSize(int segmentUpdateBatchSize) {
+ this.segmentUpdateBatchSize = segmentUpdateBatchSize;
+ }
+
public void postConstruct() {
logger.debug("postConstruct {" + bundleContext.getBundle() + "}");
loadPredefinedSegments(bundleContext);
@@ -819,64 +825,83 @@ public class SegmentServiceImpl implements SegmentService, SynchronousBundleList
long t = System.currentTimeMillis();
Condition segmentCondition = new Condition();
+ long updatedProfileCount = 0;
+
segmentCondition.setConditionType(definitionsService.getConditionType("profilePropertyCondition"));
segmentCondition.setParameter("propertyName", "segments");
segmentCondition.setParameter("comparisonOperator", "equals");
segmentCondition.setParameter("propertyValue", segment.getItemId());
if(segment.getMetadata().isEnabled()) {
- // the following list can grow really big if the segments are large.
- // We might want to replace this with scrolling if it becomes huge
- // (100million profiles)
- List<Profile> previousProfiles = persistenceService.query(segmentCondition, null, Profile.class);
- List<Profile> newProfiles = persistenceService.query(segment.getCondition(), null, Profile.class);
-
- // we use sets instead of lists to speed up contains() calls that are very expensive on lists.
-
- // we use to use removeAll calls but these are expensive because they require lots of copies upon element
- // removal so we implemented them with adds instead.
- //profilesToAdd.removeAll(previousProfiles);
- //profilesToRemove.removeAll(newProfiles);
- Set<Profile> newProfilesSet = new HashSet<>(newProfiles);
- Set<Profile> previousProfilesSet = new HashSet<>(previousProfiles);
- Set<Profile> profilesToAdd = new HashSet<>(newProfilesSet.size() / 2);
- for (Profile newProfile : newProfilesSet) {
- if (!previousProfilesSet.contains(newProfile)) {
- profilesToAdd.add(newProfile);
+ ConditionType booleanConditionType = definitionsService.getConditionType("booleanCondition");
+ ConditionType notConditionType = definitionsService.getConditionType("notCondition");
+
+ Condition profilesToAddCondition = new Condition(booleanConditionType);
+ profilesToAddCondition.setParameter("operator", "and");
+ List<Condition> profilesToAddSubConditions = new ArrayList<>();
+ profilesToAddSubConditions.add(segment.getCondition());
+ Condition notOldSegmentCondition = new Condition(notConditionType);
+ notOldSegmentCondition.setParameter("subCondition", segmentCondition);
+ profilesToAddSubConditions.add(notOldSegmentCondition);
+ profilesToAddCondition.setParameter("subConditions", profilesToAddSubConditions);
+
+ Condition profilesToRemoveCondition = new Condition(booleanConditionType);
+ profilesToRemoveCondition.setParameter("operator", "and");
+ List<Condition> profilesToRemoveSubConditions = new ArrayList<>();
+ profilesToRemoveSubConditions.add(segmentCondition);
+ Condition notNewSegmentCondition = new Condition(notConditionType);
+ notNewSegmentCondition.setParameter("subCondition", segment.getCondition());
+ profilesToRemoveSubConditions.add(notNewSegmentCondition);
+ profilesToRemoveCondition.setParameter("subConditions", profilesToRemoveSubConditions);
+
+ PartialList<Profile> profilesToRemove = persistenceService.query(profilesToRemoveCondition, null, Profile.class, 0, segmentUpdateBatchSize, "10m");
+ PartialList<Profile> profilesToAdd = persistenceService.query(profilesToAddCondition, null, Profile.class, 0, segmentUpdateBatchSize, "10m");
+
+ while (profilesToAdd.getList().size() > 0) {
+ for (Profile profileToAdd : profilesToAdd.getList()) {
+ profileToAdd.getSegments().add(segment.getItemId());
+ persistenceService.update(profileToAdd.getItemId(), null, Profile.class, "segments", profileToAdd.getSegments());
+ Event profileUpdated = new Event("profileUpdated", null, profileToAdd, null, null, profileToAdd, new Date());
+ profileUpdated.setPersistent(false);
+ eventService.send(profileUpdated);
+ updatedProfileCount++;
}
- }
- Set<Profile> profilesToRemove = new HashSet<>(previousProfilesSet.size() / 2);
- for (Profile previousProfile : previousProfilesSet) {
- if (!newProfilesSet.contains(previousProfile)) {
- profilesToRemove.add(previousProfile);
+ profilesToAdd = persistenceService.continueScrollQuery(Profile.class, profilesToAdd.getScrollIdentifier(), profilesToAdd.getScrollTimeValidity());
+ if (profilesToAdd == null || profilesToAdd.getList().size() == 0) {
+ break;
}
}
-
-
- for (Profile profileToAdd : profilesToAdd) {
- profileToAdd.getSegments().add(segment.getItemId());
- persistenceService.update(profileToAdd.getItemId(), null, Profile.class, "segments", profileToAdd.getSegments());
- Event profileUpdated = new Event("profileUpdated", null, profileToAdd, null, null, profileToAdd, new Date());
- profileUpdated.setPersistent(false);
- eventService.send(profileUpdated);
- }
- for (Profile profileToRemove : profilesToRemove) {
- profileToRemove.getSegments().remove(segment.getItemId());
- persistenceService.update(profileToRemove.getItemId(), null, Profile.class, "segments", profileToRemove.getSegments());
- Event profileUpdated = new Event("profileUpdated", null, profileToRemove, null, null, profileToRemove, new Date());
- profileUpdated.setPersistent(false);
- eventService.send(profileUpdated);
+ while (profilesToRemove.getList().size() > 0) {
+ for (Profile profileToRemove : profilesToRemove.getList()) {
+ profileToRemove.getSegments().remove(segment.getItemId());
+ persistenceService.update(profileToRemove.getItemId(), null, Profile.class, "segments", profileToRemove.getSegments());
+ Event profileUpdated = new Event("profileUpdated", null, profileToRemove, null, null, profileToRemove, new Date());
+ profileUpdated.setPersistent(false);
+ eventService.send(profileUpdated);
+ updatedProfileCount++;
+ }
+ profilesToRemove = persistenceService.continueScrollQuery(Profile.class, profilesToRemove.getScrollIdentifier(), profilesToRemove.getScrollTimeValidity());
+ if (profilesToRemove == null || profilesToRemove.getList().size() == 0) {
+ break;
+ }
}
} else {
- List<Profile> previousProfiles = persistenceService.query(segmentCondition, null, Profile.class);
- for (Profile profileToRemove : previousProfiles) {
- profileToRemove.getSegments().remove(segment.getItemId());
- persistenceService.update(profileToRemove.getItemId(), null, Profile.class, "segments", profileToRemove.getSegments());
+ PartialList<Profile> profilesToRemove = persistenceService.query(segmentCondition, null, Profile.class, 0, 200, "10m");
+ while (profilesToRemove.getList().size() > 0) {
+ for (Profile profileToRemove : profilesToRemove.getList()) {
+ profileToRemove.getSegments().remove(segment.getItemId());
+ persistenceService.update(profileToRemove.getItemId(), null, Profile.class, "segments", profileToRemove.getSegments());
+ updatedProfileCount++;
+ }
+ profilesToRemove = persistenceService.continueScrollQuery(Profile.class, profilesToRemove.getScrollIdentifier(), profilesToRemove.getScrollTimeValidity());
+ if (profilesToRemove == null || profilesToRemove.getList().size() == 0) {
+ break;
+ }
}
}
- logger.info("Profiles updated in {}ms", System.currentTimeMillis()-t);
+ logger.info("{} profiles updated in {}ms", updatedProfileCount, System.currentTimeMillis()-t);
}
private void updateExistingProfilesForScoring(Scoring scoring) {
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/28d7dbde/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
----------------------------------------------------------------------
diff --git a/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index 5564e94..3176639 100644
--- a/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ b/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -29,6 +29,7 @@
<cm:property name="profile.purge.inactiveTime" value="30"/>
<cm:property name="profile.purge.existTime" value="-1"/>
<cm:property name="event.purge.existTime" value="12"/>
+ <cm:property name="segment.update.batchSize" value="1000"/>
</cm:default-properties>
</cm:property-placeholder>
@@ -104,6 +105,7 @@
<property name="rulesService" ref="rulesServiceImpl"/>
<property name="bundleContext" ref="blueprintBundleContext"/>
<property name="taskExecutionPeriod" value="86400000"/>
+ <property name="segmentUpdateBatchSize" value="${services.segment.update.batchSize}" />
</bean>
<service id="segmentService" ref="segmentServiceImpl" auto-export="interfaces"/>
[26/50] [abbrv] incubator-unomi git commit: - Improve Geonames
startup to make it more resilient to missing mappings - Add sanity checks for
endpoints before adding them - Add database copying (Geocity and Geonames) to
test script.
Posted by sh...@apache.org.
- Improve Geonames startup to make it more resilient to missing mappings
- Add sanity checks for endpoints before adding them
- Add database copying (Geocity and Geonames) to test script.
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/603346ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/603346ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/603346ad
Branch: refs/heads/master
Commit: 603346ad3fc50756edbeeaa19e6204e7b82815fe
Parents: e6ec7a0
Author: Serge Huber <sh...@apache.org>
Authored: Mon Dec 19 16:36:41 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Mon Dec 19 16:36:41 2016 +0100
----------------------------------------------------------------------
.gitignore | 3 +
buildAndRunNoTests.sh | 8 ++
.../geonames/services/GeonamesServiceImpl.java | 85 ++++++++++++--------
.../ElasticSearchPersistenceServiceImpl.java | 25 +++---
.../persistence/spi/PersistenceService.java | 7 +-
5 files changed, 85 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/603346ad/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index efa4748..bf90830 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,6 @@ target
.DS_Store
/performance-tests/.cache
maven-metadata-local.xml
+GeoLite2-City.mmdb
+allCountries.zip
+rest/.miredot-offline.json
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/603346ad/buildAndRunNoTests.sh
----------------------------------------------------------------------
diff --git a/buildAndRunNoTests.sh b/buildAndRunNoTests.sh
index c8f7641..fa53832 100755
--- a/buildAndRunNoTests.sh
+++ b/buildAndRunNoTests.sh
@@ -27,6 +27,14 @@ mvn clean install -P \!integration-tests,\!performance-tests,rat
pushd package/target
echo Uncompressing Unomi package...
tar zxvf unomi-$UNOMI_VERSION.tar.gz
+if [ -f "../../GeoLite2-City.mmdb" ]; then
+ echo Installing GeoLite2 City database...
+ cp ../../GeoLite2-City.mmdb unomi-$UNOMI_VERSION/etc
+fi
+if [ -f "../../allCountries.zip" ]; then
+ echo Installing Geonames countries database...
+ cp ../../allCountries.zip unomi-$UNOMI_VERSION/etc
+fi
cd unomi-$UNOMI_VERSION/bin
echo Starting Unomi...
./karaf debug
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/603346ad/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java
----------------------------------------------------------------------
diff --git a/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java b/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java
index e89a630..ebe53a0 100644
--- a/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java
+++ b/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java
@@ -86,46 +86,67 @@ public class GeonamesServiceImpl implements GeonamesService {
}
final File f = new File(pathToGeonamesDatabase);
if (f.exists()) {
- Timer t = new Timer();
+ final Timer t = new Timer();
t.schedule(new TimerTask() {
@Override
public void run() {
- try {
- ZipInputStream zipInputStream = new ZipInputStream(new FileInputStream(f));
- ZipEntry zipEntry = zipInputStream.getNextEntry();
-
- BufferedReader reader = new BufferedReader(new InputStreamReader(zipInputStream, "UTF-8"));
-
- SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
- String line;
- logger.info("Starting to import geonames database ...");
- while ((line = reader.readLine()) != null) {
- String[] values = line.split("\t");
-
- if (FEATURES_CLASSES.contains(values[6])) {
- GeonameEntry geonameEntry = new GeonameEntry(values[0], values[1], values[2],
- StringUtils.isEmpty(values[4]) ? null : Double.parseDouble(values[4]),
- StringUtils.isEmpty(values[5]) ? null : Double.parseDouble(values[5]),
- values[6], values[7], values[8],
- Arrays.asList(values[9].split(",")),
- values[10], values[11], values[12], values[13],
- StringUtils.isEmpty(values[14]) ? null : Integer.parseInt(values[14]),
- StringUtils.isEmpty(values[15]) ? null : Integer.parseInt(values[15]),
- values[16], values[17],
- sdf.parse(values[18]));
-
- persistenceService.save(geonameEntry);
- }
- }
- logger.info("Geonames database imported");
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- }
+ importGeoNameDatabase(f, t);
}
}, 5000);
}
}
+ private void importGeoNameDatabase(final File f, final Timer t) {
+ Map<String,Map<String,Object>> typeMappings = persistenceService.getPropertiesMapping(GeonameEntry.ITEM_TYPE);
+ if (typeMappings == null) {
+ logger.warn("Type mappings for type {} are not yet installed, delaying import until they are ready!", GeonameEntry.ITEM_TYPE);
+ t.schedule(new TimerTask() {
+ @Override
+ public void run() {
+ importGeoNameDatabase(f, t);
+ }
+ }, 5000);
+ return;
+ }
+ try {
+
+ ZipInputStream zipInputStream = new ZipInputStream(new FileInputStream(f));
+ ZipEntry zipEntry = zipInputStream.getNextEntry(); // used to advance to the first entry in the ZipInputStream
+ BufferedReader reader = new BufferedReader(new InputStreamReader(zipInputStream, "UTF-8"));
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
+ String line;
+ logger.info("Starting to import geonames database from file {}...", f);
+ long lineCount = 0;
+ long importStartTime = System.currentTimeMillis();
+ while ((line = reader.readLine()) != null) {
+ String[] values = line.split("\t");
+
+ if (FEATURES_CLASSES.contains(values[6])) {
+ GeonameEntry geonameEntry = new GeonameEntry(values[0], values[1], values[2],
+ StringUtils.isEmpty(values[4]) ? null : Double.parseDouble(values[4]),
+ StringUtils.isEmpty(values[5]) ? null : Double.parseDouble(values[5]),
+ values[6], values[7], values[8],
+ Arrays.asList(values[9].split(",")),
+ values[10], values[11], values[12], values[13],
+ StringUtils.isEmpty(values[14]) ? null : Integer.parseInt(values[14]),
+ StringUtils.isEmpty(values[15]) ? null : Integer.parseInt(values[15]),
+ values[16], values[17],
+ sdf.parse(values[18]));
+
+ persistenceService.save(geonameEntry);
+ }
+ lineCount++;
+ if (lineCount % 1000 == 0) {
+ logger.info("{} lines imported from file {}", lineCount, f);
+ }
+ }
+ logger.info("{} lines from Geonames database file {} imported in {}ms", lineCount, f, System.currentTimeMillis()-importStartTime);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+
public List<GeonameEntry> getHierarchy(String itemId) {
return getHierarchy(persistenceService.load(itemId, GeonameEntry.class));
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/603346ad/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index f1912bd..5e806d1 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -33,7 +33,10 @@ import org.apache.unomi.api.query.DateRange;
import org.apache.unomi.api.query.IpRange;
import org.apache.unomi.api.query.NumericRange;
import org.apache.unomi.api.services.ClusterService;
-import org.apache.unomi.persistence.elasticsearch.conditions.*;
+import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder;
+import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher;
+import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator;
+import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorDispatcher;
import org.apache.unomi.persistence.spi.CustomObjectMapper;
import org.apache.unomi.persistence.spi.PersistenceService;
import org.apache.unomi.persistence.spi.aggregate.*;
@@ -1516,15 +1519,19 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
ClusterNode clusterNode = new ClusterNode();
clusterNode.setHostName(karafCellarNode.getHost());
String publicEndpoint = publicNodeEndpoints.get(karafCellarNode.getId());
- String[] publicEndpointParts = publicEndpoint.split(":");
- clusterNode.setHostAddress(publicEndpointParts[0]);
- clusterNode.setPublicPort(Integer.parseInt(publicEndpointParts[1]));
+ if (publicEndpoint != null) {
+ String[] publicEndpointParts = publicEndpoint.split(":");
+ clusterNode.setHostAddress(publicEndpointParts[0]);
+ clusterNode.setPublicPort(Integer.parseInt(publicEndpointParts[1]));
+ }
String secureEndpoint = secureNodeEndpoints.get(karafCellarNode.getId());
- String[] secureEndpointParts = secureEndpoint.split(":");
- clusterNode.setSecureHostAddress(secureEndpointParts[0]);
- clusterNode.setSecurePort(Integer.parseInt(secureEndpointParts[1]));
- clusterNode.setMaster(false);
- clusterNode.setData(false);
+ if (secureEndpoint != null) {
+ String[] secureEndpointParts = secureEndpoint.split(":");
+ clusterNode.setSecureHostAddress(secureEndpointParts[0]);
+ clusterNode.setSecurePort(Integer.parseInt(secureEndpointParts[1]));
+ clusterNode.setMaster(false);
+ clusterNode.setData(false);
+ }
try {
// now let's connect to remote JMX service to retrieve information from the runtime and operating system MX beans
JMXServiceURL url = new JMXServiceURL("service:jmx:rmi:///jndi/rmi://"+karafCellarNode.getHost() + ":"+karafJMXPort+"/karaf-root");
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/603346ad/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
----------------------------------------------------------------------
diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
index 5501645..e191829 100644
--- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
+++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
@@ -163,9 +163,12 @@ public interface PersistenceService {
boolean removeQuery(String queryName);
/**
- * TODO
+ * Retrieve the type mappings for a given itemType. This method queries the persistence service implementation
+ * to retrieve any type mappings it may have for the specified itemType.
*
- * @param itemType
+ * This method may not return any results if the implementation doesn't support property type mappings
+ *
+ * @param itemType the itemType we want to retrieve the mappings for
* @return
*/
Map<String, Map<String, Object>> getPropertiesMapping(String itemType);
[17/50] [abbrv] incubator-unomi git commit: New segment update
optimization, hopefully will work better than the last one.
Posted by sh...@apache.org.
New segment update optimization, hopefully will work better than the last one.
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/d7a79695
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/d7a79695
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/d7a79695
Branch: refs/heads/master
Commit: d7a7969578817eebf7a72c0c364200f0f084bb10
Parents: 2263da9
Author: Serge Huber <sh...@apache.org>
Authored: Tue Dec 6 13:42:54 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Tue Dec 6 13:42:54 2016 +0100
----------------------------------------------------------------------
.../services/services/SegmentServiceImpl.java | 26 +++++++++++++++++---
1 file changed, 22 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/d7a79695/services/src/main/java/org/apache/unomi/services/services/SegmentServiceImpl.java
----------------------------------------------------------------------
diff --git a/services/src/main/java/org/apache/unomi/services/services/SegmentServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/SegmentServiceImpl.java
index d1f120f..3a64be1 100644
--- a/services/src/main/java/org/apache/unomi/services/services/SegmentServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/services/SegmentServiceImpl.java
@@ -831,10 +831,28 @@ public class SegmentServiceImpl implements SegmentService, SynchronousBundleList
List<Profile> previousProfiles = persistenceService.query(segmentCondition, null, Profile.class);
List<Profile> newProfiles = persistenceService.query(segment.getCondition(), null, Profile.class);
- Set<Profile> profilesToAdd = new HashSet<>(newProfiles);
- Set<Profile> profilesToRemove = new HashSet<>(previousProfiles);
- profilesToAdd.removeAll(previousProfiles);
- profilesToRemove.removeAll(newProfiles);
+ // we use sets instead of lists to speed up contains() calls that are very expensive on lists.
+
+ // we use to use removeAll calls but these are expensive because they require lots of copies upon element
+ // removal so we implemented them with adds instead.
+ //profilesToAdd.removeAll(previousProfiles);
+ //profilesToRemove.removeAll(newProfiles);
+
+ Set<Profile> newProfilesSet = new HashSet<>(newProfiles);
+ Set<Profile> previousProfilesSet = new HashSet<>(previousProfiles);
+ Set<Profile> profilesToAdd = new HashSet<>(newProfilesSet.size() / 2);
+ for (Profile newProfile : newProfilesSet) {
+ if (!previousProfilesSet.contains(newProfile)) {
+ profilesToAdd.add(newProfile);
+ }
+ }
+ Set<Profile> profilesToRemove = new HashSet<>(previousProfilesSet.size() / 2);
+ for (Profile previousProfile : previousProfilesSet) {
+ if (!newProfilesSet.contains(previousProfile)) {
+ profilesToRemove.add(previousProfile);
+ }
+ }
+
for (Profile profileToAdd : profilesToAdd) {
profileToAdd.getSegments().add(segment.getItemId());
[48/50] [abbrv] incubator-unomi git commit: Cleanup setenv files to
remove obsolete settings
Posted by sh...@apache.org.
Cleanup setenv files to remove obsolete settings
Signed-off-by: Serge Huber <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/26f2ef2a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/26f2ef2a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/26f2ef2a
Branch: refs/heads/master
Commit: 26f2ef2a516233a296632373e592c70a602509f7
Parents: a5b7b15
Author: Serge Huber <sh...@apache.org>
Authored: Mon Jan 9 17:02:43 2017 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Mon Jan 9 17:02:43 2017 +0100
----------------------------------------------------------------------
package/src/main/resources/bin/setenv | 2 --
package/src/main/resources/bin/setenv.bat | 1 -
2 files changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/26f2ef2a/package/src/main/resources/bin/setenv
----------------------------------------------------------------------
diff --git a/package/src/main/resources/bin/setenv b/package/src/main/resources/bin/setenv
index 1b1a4df..211b6e0 100755
--- a/package/src/main/resources/bin/setenv
+++ b/package/src/main/resources/bin/setenv
@@ -49,9 +49,7 @@
MY_DIRNAME=`dirname $0`
MY_KARAF_HOME=`cd "$MY_DIRNAME/.."; pwd`
-export KARAF_OPTS="-Djava.library.path=$MY_KARAF_HOME/lib/sigar"
export JAVA_MAX_MEM=3G
-export JAVA_MAX_PERM_MEM=512M
# export YOURKIT_AGENTPATH="/Applications/YourKit_Java_Profiler_2013_build_13048.app/bin/mac/libyjpagent.jnilib"
# export KARAF_OPTS="-agentpath:$YOURKIT_AGENTPATH=disablestacktelemetry,disableexceptiontelemetry,delay=10000"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/26f2ef2a/package/src/main/resources/bin/setenv.bat
----------------------------------------------------------------------
diff --git a/package/src/main/resources/bin/setenv.bat b/package/src/main/resources/bin/setenv.bat
index 72169f8..a7ae1f4 100644
--- a/package/src/main/resources/bin/setenv.bat
+++ b/package/src/main/resources/bin/setenv.bat
@@ -63,4 +63,3 @@ rem SET KARAF_DEBUG
set MY_DIRNAME=%~dp0%
set MY_KARAF_HOME=%DIRNAME%..
-set KARAF_OPTS="-Djava.library.path=%MY_KARAF_HOME%/lib/sigar"
[36/50] [abbrv] incubator-unomi git commit: - Use linked hash map in
GeonameEntry to make sure that we conserve the order of the properties - Fix
type mapping detection algorithm
Posted by sh...@apache.org.
- Use linked hash map in GeonameEntry to make sure that we conserve the order of the properties
- Fix type mapping detection algorithm
Signed-off-by: Serge Huber <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/8d54f8d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/8d54f8d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/8d54f8d9
Branch: refs/heads/master
Commit: 8d54f8d93c45187aaf9307932f461ecd9b6e9da9
Parents: 69bb3cc
Author: Serge Huber <sh...@apache.org>
Authored: Wed Dec 21 21:27:08 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Wed Dec 21 21:27:08 2016 +0100
----------------------------------------------------------------------
.../java/org/apache/unomi/geonames/services/GeonameEntry.java | 4 ++--
.../org/apache/unomi/geonames/services/GeonamesServiceImpl.java | 4 +++-
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8d54f8d9/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonameEntry.java
----------------------------------------------------------------------
diff --git a/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonameEntry.java b/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonameEntry.java
index 014a699..ac41a6e 100644
--- a/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonameEntry.java
+++ b/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonameEntry.java
@@ -20,7 +20,7 @@ package org.apache.unomi.geonames.services;
import org.apache.unomi.api.Item;
import java.util.Date;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -56,7 +56,7 @@ public class GeonameEntry extends Item {
this.name = name;
this.asciiname = asciiname;
// this.alternatenames = alternatenames;
- this.location = new HashMap<>();
+ this.location = new LinkedHashMap<>();
this.location.put("lat",lat);
this.location.put("lon",lon);
this.featureClass = featureClass;
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8d54f8d9/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java
----------------------------------------------------------------------
diff --git a/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java b/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java
index 2b0a0dc..9bce9b2 100644
--- a/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java
+++ b/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java
@@ -98,7 +98,7 @@ public class GeonamesServiceImpl implements GeonamesService {
private void importGeoNameDatabase(final File f, final Timer t) {
Map<String,Map<String,Object>> typeMappings = persistenceService.getPropertiesMapping(GeonameEntry.ITEM_TYPE);
- if (typeMappings == null) {
+ if (typeMappings == null || typeMappings.size() == 0) {
logger.warn("Type mappings for type {} are not yet installed, delaying import until they are ready!", GeonameEntry.ITEM_TYPE);
t.schedule(new TimerTask() {
@Override
@@ -107,6 +107,8 @@ public class GeonamesServiceImpl implements GeonamesService {
}
}, 5000);
return;
+ } else {
+ // let's check that the mappings are correct
}
try {
[43/50] [abbrv] incubator-unomi git commit: UNOMI-70 : Provide
default hazelcast configuration
Posted by sh...@apache.org.
UNOMI-70 : Provide default hazelcast configuration
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/88f493cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/88f493cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/88f493cd
Branch: refs/heads/master
Commit: 88f493cd5109072792b55abd917e20656c226169
Parents: adb5fae
Author: Thomas Draier <dr...@apache.org>
Authored: Tue Jan 3 18:36:17 2017 +0100
Committer: Thomas Draier <dr...@apache.org>
Committed: Tue Jan 3 18:36:17 2017 +0100
----------------------------------------------------------------------
kar/src/main/feature/feature.xml | 1 +
package/pom.xml | 11 +
persistence-elasticsearch/core/pom.xml | 5 +
.../core/src/main/resources/hazelcast.xml | 219 +++++++++++++++++++
4 files changed, 236 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/88f493cd/kar/src/main/feature/feature.xml
----------------------------------------------------------------------
diff --git a/kar/src/main/feature/feature.xml b/kar/src/main/feature/feature.xml
index 674e6bc..8e6ddb2 100644
--- a/kar/src/main/feature/feature.xml
+++ b/kar/src/main/feature/feature.xml
@@ -32,6 +32,7 @@
<configfile finalname="/etc/org.apache.unomi.privacy.cfg">mvn:org.apache.unomi/cxs-privacy-extension-services/${project.version}/cfg/privacycfg</configfile>
<configfile finalname="/etc/org.apache.unomi.geonames.cfg">mvn:org.apache.unomi/cxs-geonames-services/${project.version}/cfg/geonamescfg</configfile>
<configfile finalname="/etc/elasticsearch.yml">mvn:org.apache.unomi/unomi-persistence-elasticsearch-core/${project.version}/yml/elasticsearchconfig</configfile>
+ <configfile finalname="/etc/hazelcast.xml">mvn:org.apache.unomi/unomi-persistence-elasticsearch-core/${project.version}/xml/hazelcastconfig</configfile>
<bundle start-level="75">mvn:commons-io/commons-io/2.4</bundle>
<bundle start-level="75">mvn:com.fasterxml.jackson.core/jackson-core/${version.jackson.core}</bundle>
<bundle start-level="75">mvn:com.fasterxml.jackson.core/jackson-databind/${version.jackson.core}</bundle>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/88f493cd/package/pom.xml
----------------------------------------------------------------------
diff --git a/package/pom.xml b/package/pom.xml
index 6f622d9..104723e 100644
--- a/package/pom.xml
+++ b/package/pom.xml
@@ -226,6 +226,17 @@
</artifactItem>
<artifactItem>
<groupId>org.apache.unomi</groupId>
+ <artifactId>unomi-persistence-elasticsearch-core</artifactId>
+ <version>${project.version}</version>
+ <classifier>hazelcastconfig</classifier>
+ <type>xml</type>
+ <outputDirectory>
+ ${project.build.directory}/assembly/etc
+ </outputDirectory>
+ <destFileName>hazelcast.xml</destFileName>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins-request</artifactId>
<version>${project.version}</version>
<classifier>requestcfg</classifier>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/88f493cd/persistence-elasticsearch/core/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/pom.xml b/persistence-elasticsearch/core/pom.xml
index 6a05775..f4b2a24 100644
--- a/persistence-elasticsearch/core/pom.xml
+++ b/persistence-elasticsearch/core/pom.xml
@@ -260,6 +260,11 @@
<type>yml</type>
<classifier>elasticsearchconfig</classifier>
</artifact>
+ <artifact>
+ <file>src/main/resources/hazelcast.xml</file>
+ <type>xml</type>
+ <classifier>hazelcastconfig</classifier>
+ </artifact>
</artifacts>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/88f493cd/persistence-elasticsearch/core/src/main/resources/hazelcast.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/hazelcast.xml b/persistence-elasticsearch/core/src/main/resources/hazelcast.xml
new file mode 100644
index 0000000..0fc6f5d
--- /dev/null
+++ b/persistence-elasticsearch/core/src/main/resources/hazelcast.xml
@@ -0,0 +1,219 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<hazelcast xsi:schemaLocation="http://www.hazelcast.com/schema/config hazelcast-config-3.2.xsd"
+ xmlns="http://www.hazelcast.com/schema/config"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <group>
+ <name>cellar</name>
+ <password>pass</password>
+ </group>
+ <management-center enabled="false">http://localhost:8080/mancenter</management-center>
+ <network>
+ <port auto-increment="true" port-count="100">5701</port>
+ <outbound-ports>
+ <!--
+ Allowed port range when connecting to other nodes.
+ 0 or * means use system provided port.
+ -->
+ <ports>0</ports>
+ </outbound-ports>
+ <join>
+ <multicast enabled="false">
+ <multicast-group>224.2.2.3</multicast-group>
+ <multicast-port>54327</multicast-port>
+ </multicast>
+ <tcp-ip enabled="true">
+ <interface>127.0.0.1</interface>
+ </tcp-ip>
+ <aws enabled="false">
+ <access-key>my-access-key</access-key>
+ <secret-key>my-secret-key</secret-key>
+ <!--optional, default is us-east-1 -->
+ <region>us-west-1</region>
+ <!--optional, default is ec2.amazonaws.com. If set, region shouldn't be set as it will override this property -->
+ <host-header>ec2.amazonaws.com</host-header>
+ <!-- optional, only instances belonging to this group will be discovered, default will try all running instances -->
+ <security-group-name>hazelcast-sg</security-group-name>
+ <tag-key>type</tag-key>
+ <tag-value>hz-nodes</tag-value>
+ </aws>
+ </join>
+ <interfaces enabled="false">
+ <interface>10.10.1.*</interface>
+ </interfaces>
+ <ssl enabled="false"/>
+ <socket-interceptor enabled="false"/>
+ <symmetric-encryption enabled="false">
+ <!--
+ encryption algorithm such as
+ DES/ECB/PKCS5Padding,
+ PBEWithMD5AndDES,
+ AES/CBC/PKCS5Padding,
+ Blowfish,
+ DESede
+ -->
+ <algorithm>PBEWithMD5AndDES</algorithm>
+ <!-- salt value to use when generating the secret key -->
+ <salt>thesalt</salt>
+ <!-- pass phrase to use when generating the secret key -->
+ <password>thepass</password>
+ <!-- iteration count to use when generating the secret key -->
+ <iteration-count>19</iteration-count>
+ </symmetric-encryption>
+ </network>
+ <partition-group enabled="false"/>
+ <executor-service>
+ <pool-size>16</pool-size>
+ <!-- Queue capacity. 0 means Integer.MAX_VALUE -->
+ <queue-capacity>0</queue-capacity>
+ </executor-service>
+ <queue name="default">
+ <!--
+ Maximum size of the queue. When a JVM's local queue size reaches the maximum,
+ all put/offer operations will get blocked until the queue size
+ of the JVM goes down below the maximum.
+ Any integer between 0 and Integer.MAX_VALUE. 0 means
+ Integer.MAX_VALUE. Default is 0.
+ -->
+ <max-size>0</max-size>
+ <!--
+ Number of backups. If 1 is set as the backup-count for example,
+ then all entries of the map will be copied to another JVM for
+ fail-safety. 0 means no backup.
+ -->
+ <backup-count>1</backup-count>
+ <!--
+ Number of async backups. 0 means no backup.
+ -->
+ <async-backup-count>0</async-backup-count>
+ <empty-queue-ttl>-1</empty-queue-ttl>
+ </queue>
+
+ <map name="default">
+ <!--
+ Data type that will be used for storing recordMap.
+ Possible values:
+ BINARY (default): keys and values will be stored as binary data
+ OBJECT : values will be stored in their object forms
+ OFFHEAP : values will be stored in non-heap region of JVM
+ -->
+ <in-memory-format>BINARY</in-memory-format>
+ <!--
+ Number of backups. If 1 is set as the backup-count for example,
+ then all entries of the map will be copied to another JVM for
+ fail-safety. 0 means no backup.
+ -->
+ <backup-count>1</backup-count>
+ <!--
+ Number of async backups. 0 means no backup.
+ -->
+ <async-backup-count>0</async-backup-count>
+ <!--
+ Maximum number of seconds for each entry to stay in the map. Entries that are
+ older than <time-to-live-seconds> and not updated for <time-to-live-seconds>
+ will get automatically evicted from the map.
+ Any integer between 0 and Integer.MAX_VALUE. 0 means infinite. Default is 0.
+ -->
+ <time-to-live-seconds>0</time-to-live-seconds>
+ <!--
+ Maximum number of seconds for each entry to stay idle in the map. Entries that are
+ idle(not touched) for more than <max-idle-seconds> will get
+ automatically evicted from the map. Entry is touched if get, put or containsKey is called.
+ Any integer between 0 and Integer.MAX_VALUE. 0 means infinite. Default is 0.
+ -->
+ <max-idle-seconds>0</max-idle-seconds>
+ <!--
+ Valid values are:
+ NONE (no eviction),
+ LRU (Least Recently Used),
+ LFU (Least Frequently Used).
+ NONE is the default.
+ -->
+ <eviction-policy>NONE</eviction-policy>
+ <!--
+ Maximum size of the map. When max size is reached,
+ map is evicted based on the policy defined.
+ Any integer between 0 and Integer.MAX_VALUE. 0 means
+ Integer.MAX_VALUE. Default is 0.
+ -->
+ <max-size policy="PER_NODE">0</max-size>
+ <!--
+ When max. size is reached, specified percentage of
+ the map will be evicted. Any integer between 0 and 100.
+ If 25 is set for example, 25% of the entries will
+ get evicted.
+ -->
+ <eviction-percentage>25</eviction-percentage>
+ <!--
+ While recovering from split-brain (network partitioning),
+ map entries in the small cluster will merge into the bigger cluster
+ based on the policy set here. When an entry merge into the
+ cluster, there might an existing entry with the same key already.
+ Values of these entries might be different for that same key.
+ Which value should be set for the key? Conflict is resolved by
+ the policy set here. Default policy is PutIfAbsentMapMergePolicy
+
+ There are built-in merge policies such as
+ com.hazelcast.map.merge.PassThroughMergePolicy; entry will be added if there is no existing entry for the key.
+ com.hazelcast.map.merge.PutIfAbsentMapMergePolicy ; entry will be added if the merging entry doesn't exist in the cluster.
+ com.hazelcast.map.merge.HigherHitsMapMergePolicy ; entry with the higher hits wins.
+ com.hazelcast.map.merge.LatestUpdateMapMergePolicy ; entry with the latest update wins.
+ -->
+ <merge-policy>com.hazelcast.map.merge.PassThroughMergePolicy</merge-policy>
+ </map>
+
+ <multimap name="default">
+ <backup-count>1</backup-count>
+ <value-collection-type>SET</value-collection-type>
+ </multimap>
+
+ <multimap name="default">
+ <backup-count>1</backup-count>
+ <value-collection-type>SET</value-collection-type>
+ </multimap>
+
+ <list name="default">
+ <backup-count>1</backup-count>
+ </list>
+
+ <set name="default">
+ <backup-count>1</backup-count>
+ </set>
+
+ <jobtracker name="default">
+ <max-thread-size>0</max-thread-size>
+ <!-- Queue size 0 means number of partitions * 2 -->
+ <queue-size>0</queue-size>
+ <retry-count>0</retry-count>
+ <chunk-size>1000</chunk-size>
+ <communicate-stats>true</communicate-stats>
+ <topology-changed-strategy>CANCEL_RUNNING_OPERATION</topology-changed-strategy>
+ </jobtracker>
+
+ <semaphore name="default">
+ <initial-permits>0</initial-permits>
+ <backup-count>1</backup-count>
+ <async-backup-count>0</async-backup-count>
+ </semaphore>
+
+ <serialization>
+ <portable-version>0</portable-version>
+ </serialization>
+
+ <services enable-defaults="true" />
+</hazelcast>
\ No newline at end of file
[25/50] [abbrv] incubator-unomi git commit: UNOMI-70 : updated
mappings, fixed aggregation and queries
Posted by sh...@apache.org.
UNOMI-70 : updated mappings, fixed aggregation and queries
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/e6ec7a04
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/e6ec7a04
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/e6ec7a04
Branch: refs/heads/master
Commit: e6ec7a0425748a4fe7c0b4a05b63dfede2cc12c4
Parents: ca5c990
Author: Thomas Draier <dr...@apache.org>
Authored: Mon Dec 19 10:58:31 2016 +0100
Committer: Thomas Draier <dr...@apache.org>
Committed: Mon Dec 19 15:10:14 2016 +0100
----------------------------------------------------------------------
.../META-INF/cxs/mappings/userList.json | 22 +-
.../ElasticSearchPersistenceServiceImpl.java | 20 +-
.../META-INF/cxs/mappings/actionType.json | 16 -
.../META-INF/cxs/mappings/campaign.json | 28 +-
.../META-INF/cxs/mappings/campaignevent.json | 25 +-
.../META-INF/cxs/mappings/conditionType.json | 76 -----
.../resources/META-INF/cxs/mappings/event.json | 312 +------------------
.../resources/META-INF/cxs/mappings/goal.json | 22 +-
.../META-INF/cxs/mappings/persona.json | 32 --
.../META-INF/cxs/mappings/personaSession.json | 56 ----
.../META-INF/cxs/mappings/profile.json | 27 +-
.../META-INF/cxs/mappings/propertyType.json | 24 +-
.../resources/META-INF/cxs/mappings/rule.json | 19 --
.../META-INF/cxs/mappings/scoring.json | 22 +-
.../META-INF/cxs/mappings/segment.json | 22 +-
.../META-INF/cxs/mappings/session.json | 24 +-
.../PropertyConditionESQueryBuilder.java | 4 +-
.../conditions/PropertyConditionEvaluator.java | 12 +-
.../services/services/GoalsServiceImpl.java | 2 +-
.../services/services/QueryServiceImpl.java | 2 +-
20 files changed, 47 insertions(+), 720 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json
----------------------------------------------------------------------
diff --git a/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json b/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json
index 244d71a..53fd043 100644
--- a/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json
+++ b/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json
@@ -1,17 +1,5 @@
{
"userList": {
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "text",
- "mapping": {
- "type": "text",
- "analyzer" : "folding"
- }
- }
- }
- ],
"properties": {
"itemId": {
"type": "keyword"
@@ -21,10 +9,6 @@
},
"metadata": {
"properties": {
- "description": {
- "type": "text",
- "analyzer": "folding"
- },
"enabled": {
"type": "boolean"
},
@@ -37,14 +21,14 @@
"missingPlugins": {
"type": "boolean"
},
- "name": {
- "type": "keyword"
- },
"readOnly": {
"type": "boolean"
},
"scope": {
"type": "keyword"
+ },
+ "tags": {
+ "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 020d1d4..f1912bd 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -1135,12 +1135,12 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
@Override
public <T extends Item> List<T> query(final String fieldName, final String[] fieldValues, String sortBy, final Class<T> clazz) {
- return query(QueryBuilders.termsQuery(fieldName, ConditionContextHelper.foldToASCII(fieldValues)), sortBy, clazz, 0, -1, getRouting(fieldName, fieldValues, clazz), null).getList();
+ return query(QueryBuilders.termsQuery(fieldName, fieldValues), sortBy, clazz, 0, -1, getRouting(fieldName, fieldValues, clazz), null).getList();
}
@Override
public <T extends Item> PartialList<T> query(String fieldName, String fieldValue, String sortBy, Class<T> clazz, int offset, int size) {
- return query(QueryBuilders.termQuery(fieldName, ConditionContextHelper.foldToASCII(fieldValue)), sortBy, clazz, offset, size, getRouting(fieldName, new String[]{fieldValue}, clazz), null);
+ return query(QueryBuilders.termQuery(fieldName, fieldValue), sortBy, clazz, offset, size, getRouting(fieldName, new String[]{fieldValue}, clazz), null);
}
@Override
@@ -1243,12 +1243,16 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
requestBuilder = requestBuilder.addSort(distanceSortBuilder.order(SortOrder.ASC));
}
} else {
+ String name = StringUtils.substringBeforeLast(sortByElement,":");
+ if (name.equals("metadata.name") || name.equals("metadata.description")
+ || name.equals("properties.firstName") || name.equals("properties.lastName")
+ ) {
+ name += ".keyword";
+ }
if (sortByElement.endsWith(":desc")) {
- requestBuilder = requestBuilder.addSort(sortByElement.substring(0, sortByElement.length() - ":desc".length()), SortOrder.DESC);
- } else if (sortByElement.endsWith(":asc")) {
- requestBuilder = requestBuilder.addSort(sortByElement.substring(0, sortByElement.length() - ":asc".length()), SortOrder.ASC);
+ requestBuilder = requestBuilder.addSort(name, SortOrder.DESC);
} else {
- requestBuilder = requestBuilder.addSort(sortByElement, SortOrder.ASC);
+ requestBuilder = requestBuilder.addSort(name, SortOrder.ASC);
}
}
}
@@ -1384,7 +1388,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
for (DateRange range : dateRangeAggregate.getDateRanges()) {
if (range != null) {
- rangebuilder.addRange(range.getKey(), range.getFrom().toString(), range.getTo().toString());
+ rangebuilder.addRange(range.getKey(), range.getFrom() != null ? range.getFrom().toString() : null, range.getTo() != null ? range.getTo().toString() : null);
}
}
bucketsAggregation = rangebuilder;
@@ -1636,7 +1640,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
new InClassLoaderExecute<Void>() {
@Override
protected Void execute(Object... args) {
- QueryBuilder query = QueryBuilders.termQuery("scope", ConditionContextHelper.foldToASCII(scope));
+ QueryBuilder query = QueryBuilders.termQuery("scope", scope);
BulkRequestBuilder deleteByScope = client.prepareBulk();
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/actionType.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/actionType.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/actionType.json
deleted file mode 100644
index fa2f0ea..0000000
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/actionType.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "actionType": {
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "text",
- "mapping": {
- "type": "text",
- "analyzer": "folding"
- }
- }
- }
- ]
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
index efdcb3c..acd4b85 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
@@ -1,17 +1,5 @@
{
"campaign": {
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "text",
- "mapping": {
- "type": "text",
- "analyzer": "folding"
- }
- }
- }
- ],
"properties": {
"cost": {
"type": "double"
@@ -20,12 +8,10 @@
"type": "keyword"
},
"startDate": {
- "type": "date",
- "format": "strict_date_optional_time||epoch_millis"
+ "type": "date"
},
"endDate": {
- "type": "date",
- "format": "strict_date_optional_time||epoch_millis"
+ "type": "date"
},
"itemId": {
"type": "keyword"
@@ -41,10 +27,6 @@
},
"metadata": {
"properties": {
- "description": {
- "type": "text",
- "analyzer": "folding"
- },
"enabled": {
"type": "boolean"
},
@@ -57,9 +39,6 @@
"missingPlugins": {
"type": "boolean"
},
- "name": {
- "type": "keyword"
- },
"readOnly": {
"type": "boolean"
},
@@ -67,8 +46,7 @@
"type": "keyword"
},
"tags": {
- "type": "text",
- "analyzer": "folding"
+ "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
index f682e6a..7308995 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
@@ -1,17 +1,5 @@
{
"campaignevent": {
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "text",
- "mapping": {
- "type": "text",
- "analyzer": "folding"
- }
- }
- }
- ],
"properties": {
"campaignId": {
"type": "keyword"
@@ -23,8 +11,7 @@
"type": "keyword"
},
"eventDate": {
- "type": "date",
- "format": "strict_date_optional_time||epoch_millis"
+ "type": "date"
},
"itemId": {
"type": "keyword"
@@ -37,10 +24,6 @@
},
"metadata": {
"properties": {
- "description": {
- "type": "text",
- "analyzer": "folding"
- },
"enabled": {
"type": "boolean"
},
@@ -53,14 +36,14 @@
"missingPlugins": {
"type": "boolean"
},
- "name": {
- "type": "keyword"
- },
"readOnly": {
"type": "boolean"
},
"scope": {
"type": "keyword"
+ },
+ "tags": {
+ "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/conditionType.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/conditionType.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/conditionType.json
deleted file mode 100644
index 702a9f1..0000000
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/conditionType.json
+++ /dev/null
@@ -1,76 +0,0 @@
-{
- "conditionType": {
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "text",
- "mapping": {
- "type": "text",
- "analyzer": "folding"
- }
- }
- }
- ],
- "properties" : {
- "conditionEvaluator" : {
- "type" : "keyword"
- },
- "itemId" : {
- "type" : "keyword"
- },
- "itemType" : {
- "type" : "keyword"
- },
- "metadata" : {
- "properties" : {
- "description" : {
- "type" : "text",
- "analyzer" : "folding"
- },
- "enabled" : {
- "type" : "boolean"
- },
- "hidden" : {
- "type" : "boolean"
- },
- "id" : {
- "type" : "keyword"
- },
- "missingPlugins" : {
- "type" : "boolean"
- },
- "name" : {
- "type" : "keyword"
- },
- "readOnly" : {
- "type" : "boolean"
- },
- "tags" : {
- "type" : "text",
- "analyzer" : "folding"
- }
- }
- },
- "parameters" : {
- "properties" : {
- "defaultValue" : {
- "type" : "keyword"
- },
- "id" : {
- "type" : "keyword"
- },
- "multivalued" : {
- "type" : "boolean"
- },
- "type" : {
- "type" : "keyword"
- }
- }
- },
- "queryBuilder" : {
- "type" : "keyword"
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
index 939120d..c240b3b 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
@@ -1,17 +1,5 @@
{
"event": {
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "text",
- "mapping": {
- "type": "text",
- "analyzer": "folding"
- }
- }
- }
- ],
"properties": {
"eventType": {
"type": "keyword"
@@ -25,17 +13,6 @@
"profileId": {
"type": "keyword"
},
- "properties": {
- "properties": {
- "src_terms[0]": {
- "properties": {
- "term": {
- "type": "keyword"
- }
- }
- }
- }
- },
"scope": {
"type": "keyword"
},
@@ -43,296 +20,9 @@
"type": "keyword"
},
"timeStamp": {
- "type": "date",
- "format": "strict_date_optional_time||epoch_millis"
- },
- "source": {
- "properties": {
- "eventType": {
- "type": "keyword"
- },
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
- "profileId": {
- "type": "keyword"
- },
- "properties": {
- "properties": {
- "pageInfo": {
- "properties": {
- "destinationURL": {
- "type": "keyword"
- },
- "language": {
- "type": "keyword"
- },
- "pageName": {
- "type": "keyword"
- },
- "pagePath": {
- "type": "keyword"
- },
- "referringURL": {
- "type": "keyword"
- },
- "variant": {
- "type": "keyword"
- }
- }
- }
- }
- },
- "scope": {
- "type": "keyword"
- },
- "sessionId": {
- "type": "keyword"
- },
- "source": {
- "properties": {
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
- "scope": {
- "type": "keyword"
- }
- }
- },
- "target": {
- "properties": {
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
- "properties": {
- "properties": {
- "pageInfo": {
- "properties": {
- "destinationURL": {
- "type": "keyword"
- },
- "language": {
- "type": "keyword"
- },
- "pageName": {
- "type": "keyword"
- },
- "pagePath": {
- "type": "keyword"
- },
- "referringURL": {
- "type": "keyword"
- },
- "variant": {
- "type": "keyword"
- }
- }
- }
- }
- },
- "scope": {
- "type": "keyword"
- }
- }
- },
- "timeStamp": {
- "type": "date"
- }
- }
- },
- "target": {
- "properties": {
- "campaignId": {
- "type": "keyword"
- },
- "duration": {
- "type": "long"
- },
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
- "metadata": {
- "properties": {
- "enabled": {
- "type": "boolean"
- },
- "hidden": {
- "type": "boolean"
- },
- "id": {
- "type": "keyword"
- },
- "missingPlugins": {
- "type": "boolean"
- },
- "name": {
- "type": "keyword"
- },
- "readOnly": {
- "type": "boolean"
- },
- "scope": {
- "type": "keyword"
- },
- "tags": {
- "type": "text",
- "analyzer": "folding",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- },
- "profile": {
- "properties": {
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
- "properties": {
- "properties": {
- "firstVisit": {
- "type": "date"
- }
- }
- }
- }
- },
- "profileId": {
- "type": "keyword"
- },
- "properties": {
- "properties": {
- "birthDate": {
- "type": "date"
- },
- "email": {
- "type": "keyword"
- },
- "faceBookId": {
- "type": "keyword"
- },
- "firstName": {
- "type": "keyword"
- },
- "gender": {
- "type": "keyword"
- },
- "goalId": {
- "type": "keyword"
- },
- "lastName": {
- "type": "keyword"
- },
- "leadAssignedTo": {
- "type": "keyword"
- },
- "nationality": {
- "type": "keyword"
- },
- "origin": {
- "type": "keyword"
- },
- "pageInfo": {
- "properties": {
- "destinationURL": {
- "type": "keyword"
- },
- "language": {
- "type": "keyword"
- },
- "pageName": {
- "type": "keyword"
- },
- "pagePath": {
- "type": "keyword"
- },
- "referringURL": {
- "type": "keyword"
- },
- "variant": {
- "type": "keyword"
- }
- }
- },
- "path": {
- "type": "keyword"
- },
- "phoneNumber": {
- "type": "keyword"
- },
- "randomNumber": {
- "type": "long"
- },
- "twitterId": {
- "type": "keyword"
- },
- "variantId": {
- "type": "keyword"
- },
- "workspace": {
- "type": "keyword"
- }
- }
- },
- "scope": {
- "type": "keyword"
- },
- "size": {
- "type": "long"
- },
- "startEvent": {
- "properties": {
- "parameterValues": {
- "properties": {
- "pagePath": {
- "type": "keyword"
- }
- }
- },
- "type": {
- "type": "keyword"
- }
- }
- },
- "targetEvent": {
- "properties": {
- "parameterValues": {
- "properties": {
- "pagePath": {
- "type": "keyword"
- }
- }
- },
- "type": {
- "type": "keyword"
- }
- }
- },
- "timeStamp": {
- "type": "date"
- }
- }
+ "type": "date"
}
}
}
}
-
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
index 7290220..081915b 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
@@ -1,17 +1,5 @@
{
"goal": {
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "text",
- "mapping": {
- "type": "text",
- "analyzer": "folding"
- }
- }
- }
- ],
"properties": {
"campaignId": {
"type": "keyword"
@@ -24,10 +12,6 @@
},
"metadata": {
"properties": {
- "description": {
- "type": "text",
- "analyzer" : "folding"
- },
"enabled": {
"type": "boolean"
},
@@ -40,9 +24,6 @@
"missingPlugins": {
"type": "boolean"
},
- "name": {
- "type": "keyword"
- },
"readOnly": {
"type": "boolean"
},
@@ -50,8 +31,7 @@
"type": "keyword"
},
"tags": {
- "type": "text",
- "analyzer": "folding"
+ "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/persona.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/persona.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/persona.json
deleted file mode 100644
index db9828b..0000000
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/persona.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
- "persona": {
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "text",
- "mapping": {
- "type": "text",
- "analyzer": "folding"
- }
- }
- }
- ],
- "properties" : {
- "properties" : {
- "properties": {
- "description": {
- "type": "text",
- "analyzer": "folding"
- },
- "firstName": {
- "type": "keyword"
- },
- "lastName": {
- "type": "keyword"
- }
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
index c54a860..cb26f1e 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
@@ -1,17 +1,5 @@
{
"personaSession": {
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "text",
- "mapping": {
- "type": "text",
- "analyzer": "folding"
- }
- }
- }
- ],
"properties": {
"duration" : {
"type" : "long"
@@ -35,20 +23,6 @@
},
"itemType" : {
"type" : "keyword"
- },
- "properties" : {
- "properties" : {
- "description" : {
- "type" : "text",
- "analyzer" : "folding"
- },
- "firstName" : {
- "type" : "keyword"
- },
- "lastName" : {
- "type" : "keyword"
- }
- }
}
}
},
@@ -57,38 +31,8 @@
},
"properties" : {
"properties" : {
- "deviceCategory" : {
- "type" : "keyword"
- },
"location" : {
"type" : "geo_point"
- },
- "operatingSystemFamily" : {
- "type" : "keyword"
- },
- "operatingSystemName" : {
- "type" : "keyword"
- },
- "remoteAddr" : {
- "type" : "keyword"
- },
- "remoteHost" : {
- "type" : "keyword"
- },
- "sessionCity" : {
- "type" : "keyword"
- },
- "sessionCountryCode" : {
- "type" : "keyword"
- },
- "sessionCountryName" : {
- "type" : "keyword"
- },
- "userAgentName" : {
- "type" : "keyword"
- },
- "userAgentVersion" : {
- "type" : "keyword"
}
}
},
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
index 9aba5d0..665941d 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
@@ -1,17 +1,5 @@
{
"profile": {
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "text",
- "mapping": {
- "type": "text",
- "analyzer": "folding"
- }
- }
- }
- ],
"properties": {
"itemId": {
"type": "keyword"
@@ -21,23 +9,14 @@
},
"properties": {
"properties": {
- "j:nodename": {
- "type": "keyword"
- },
- "preferredLanguage": {
- "type": "keyword"
- },
"firstVisit": {
- "type": "date",
- "format": "strict_date_optional_time||epoch_millis"
+ "type": "date"
},
"lastVisit": {
- "type": "date",
- "format": "strict_date_optional_time||epoch_millis"
+ "type": "date"
},
"previousVisit": {
- "type": "date",
- "format": "strict_date_optional_time||epoch_millis"
+ "type": "date"
},
"nbOfVisits": {
"type": "long"
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
index c03111c..bdba719 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
@@ -1,17 +1,5 @@
{
"propertyType": {
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "text",
- "mapping": {
- "type": "text",
- "analyzer": "folding"
- }
- }
- }
- ],
"properties": {
"automaticMappingsFrom": {
"type": "keyword"
@@ -55,11 +43,14 @@
"missingPlugins": {
"type": "boolean"
},
- "name": {
- "type": "keyword"
- },
"readOnly": {
"type": "boolean"
+ },
+ "scope": {
+ "type": "keyword"
+ },
+ "tags": {
+ "type": "keyword"
}
}
},
@@ -86,8 +77,7 @@
"type": "double"
},
"tags": {
- "type": "text",
- "analyzer": "folding"
+ "type": "keyword"
},
"target": {
"type": "keyword"
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
index 842181a..aad6029 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
@@ -1,17 +1,5 @@
{
"rule": {
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "text",
- "mapping": {
- "type": "text",
- "analyzer": "folding"
- }
- }
- }
- ],
"properties": {
"itemId": {
"type": "keyword"
@@ -21,10 +9,6 @@
},
"metadata": {
"properties": {
- "description": {
- "type": "text",
- "analyzer": "folding"
- },
"enabled": {
"type": "boolean"
},
@@ -37,9 +21,6 @@
"missingPlugins": {
"type": "boolean"
},
- "name": {
- "type": "keyword"
- },
"readOnly": {
"type": "boolean"
},
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
index 70b2f8d..b46d7e2 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
@@ -1,17 +1,5 @@
{
"scoring": {
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "text",
- "mapping": {
- "type": "text",
- "analyzer": "folding"
- }
- }
- }
- ],
"properties": {
"itemId": {
"type": "keyword"
@@ -21,10 +9,6 @@
},
"metadata": {
"properties": {
- "description": {
- "type": "text",
- "analyzer": "folding"
- },
"enabled": {
"type": "boolean"
},
@@ -37,9 +21,6 @@
"missingPlugins": {
"type": "boolean"
},
- "name": {
- "type": "keyword"
- },
"readOnly": {
"type": "boolean"
},
@@ -47,8 +28,7 @@
"type": "keyword"
},
"tags": {
- "type": "text",
- "analyzer" : "folding"
+ "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
index 2d25939..322ff19 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
@@ -1,17 +1,5 @@
{
"segment": {
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "text",
- "mapping": {
- "type": "text",
- "analyzer": "folding"
- }
- }
- }
- ],
"properties": {
"itemId": {
"type": "keyword"
@@ -21,10 +9,6 @@
},
"metadata": {
"properties": {
- "description": {
- "type": "text",
- "analyzer": "folding"
- },
"enabled": {
"type": "boolean"
},
@@ -37,14 +21,14 @@
"missingPlugins": {
"type": "boolean"
},
- "name": {
- "type": "keyword"
- },
"readOnly": {
"type": "boolean"
},
"scope": {
"type": "keyword"
+ },
+ "tags": {
+ "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
index b0f6215..b1759a3 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
@@ -1,17 +1,5 @@
{
"session": {
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "text",
- "mapping": {
- "type": "text",
- "analyzer": "folding"
- }
- }
- }
- ],
"properties": {
"duration": {
"type": "long"
@@ -23,8 +11,7 @@
"type": "keyword"
},
"lastEventDate": {
- "type": "date",
- "format": "strict_date_optional_time||epoch_millis"
+ "type": "date"
},
"profileId": {
"type": "keyword"
@@ -43,12 +30,6 @@
"properties": {
"location": {
"type": "geo_point"
- },
- "pageReferringURL": {
- "type": "keyword"
- },
- "referringURL": {
- "type": "keyword"
}
}
},
@@ -59,8 +40,7 @@
"type": "long"
},
"timeStamp": {
- "type": "date",
- "format": "strict_date_optional_time||epoch_millis"
+ "type": "date"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
index c36722b..31aa0a3 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
@@ -44,12 +44,12 @@ public class PropertyConditionESQueryBuilder implements ConditionESQueryBuilder
throw new IllegalArgumentException("Impossible to build ES filter, condition is not valid, comparisonOperator and propertyName properties should be provided");
}
- String expectedValue = ConditionContextHelper.foldToASCII((String) condition.getParameter("propertyValue"));
+ String expectedValue = (String) condition.getParameter("propertyValue");
Object expectedValueInteger = condition.getParameter("propertyValueInteger");
Object expectedValueDate = condition.getParameter("propertyValueDate");
Object expectedValueDateExpr = condition.getParameter("propertyValueDateExpr");
- List<?> expectedValues = ConditionContextHelper.foldToASCII((List<?>) condition.getParameter("propertyValues"));
+ List<?> expectedValues = (List<?>) condition.getParameter("propertyValues");
List<?> expectedValuesInteger = (List<?>) condition.getParameter("propertyValuesInteger");
List<?> expectedValuesDate = (List<?>) condition.getParameter("propertyValuesDate");
List<?> expectedValuesDateExpr = (List<?>) condition.getParameter("propertyValuesDateExpr");
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
index 68e58a6..d8b7754 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
@@ -83,7 +83,7 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
return false;
}
- List<Object> actual = ConditionContextHelper.foldToASCII(getValueSet(actualValue));
+ List<Object> actual = getValueSet(actualValue);
boolean result = true;
@@ -136,7 +136,7 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
String op = (String) condition.getParameter("comparisonOperator");
String name = (String) condition.getParameter("propertyName");
- String expectedValue = ConditionContextHelper.foldToASCII((String) condition.getParameter("propertyValue"));
+ String expectedValue = (String) condition.getParameter("propertyValue");
Object expectedValueInteger = condition.getParameter("propertyValueInteger");
Object expectedValueDate = condition.getParameter("propertyValueDate");
Object expectedValueDateExpr = condition.getParameter("propertyValueDateExpr");
@@ -165,9 +165,6 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
actualValue = null;
}
}
- if (actualValue instanceof String) {
- actualValue = ConditionContextHelper.foldToASCII((String) actualValue);
- }
if(op == null) {
return false;
@@ -178,9 +175,6 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
} else if (op.equals("equals")) {
if (actualValue instanceof Collection) {
for (Object o : ((Collection<?>)actualValue)) {
- if (o instanceof String) {
- o = ConditionContextHelper.foldToASCII((String) o);
- }
if (compare(o, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr) == 0) {
return true;
}
@@ -220,7 +214,7 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
} else if (op.equals("matchesRegex")) {
return expectedValue != null && Pattern.compile(expectedValue).matcher(actualValue.toString()).matches();
} else if (op.equals("in") || op.equals("notIn") || op.equals("hasSomeOf") || op.equals("hasNoneOf") || op.equals("all")) {
- List<?> expectedValues = ConditionContextHelper.foldToASCII((List<?>) condition.getParameter("propertyValues"));
+ List<?> expectedValues = (List<?>) condition.getParameter("propertyValues");
List<?> expectedValuesInteger = (List<?>) condition.getParameter("propertyValuesInteger");
List<?> expectedValuesDate = (List<?>) condition.getParameter("propertyValuesDate");
List<?> expectedValuesDateExpr = (List<?>) condition.getParameter("propertyValuesDateExpr");
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java
----------------------------------------------------------------------
diff --git a/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java
index 65553fb..a99f155 100644
--- a/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java
@@ -492,7 +492,7 @@ public class GoalsServiceImpl implements GoalsService, SynchronousBundleListener
}
if(aggregate == null){
- aggregate = new TermsAggregate(query.getAggregate().getProperty());
+ aggregate = new TermsAggregate(query.getAggregate().getProperty() + ".keyword");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/e6ec7a04/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
----------------------------------------------------------------------
diff --git a/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
index 33e4f47..79edd43 100644
--- a/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
@@ -83,7 +83,7 @@ public class QueryServiceImpl implements QueryService {
}
// fall back on terms aggregate
- return persistenceService.aggregateQuery(query.getCondition(), new TermsAggregate(property), itemType);
+ return persistenceService.aggregateQuery(query.getCondition(), new TermsAggregate(property + ".keyword"), itemType);
}
return getAggregate(itemType, property);
[40/50] [abbrv] incubator-unomi git commit: Make condition evaluator
and condition ES query builders registration more solid by using reference
listeners.
Posted by sh...@apache.org.
Make condition evaluator and condition ES query builders registration more solid by using reference listeners.
Signed-off-by: Serge Huber <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/1c11ecc1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/1c11ecc1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/1c11ecc1
Branch: refs/heads/master
Commit: 1c11ecc109469a6c7b5bc446d4b35d4eeb82bee6
Parents: 2ff9286
Author: Serge Huber <sh...@apache.org>
Authored: Thu Dec 22 10:42:42 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Thu Dec 22 10:42:42 2016 +0100
----------------------------------------------------------------------
.../ElasticSearchPersistenceServiceImpl.java | 49 +++++++-------------
.../ConditionESQueryBuilderDispatcher.java | 10 ++++
.../ConditionEvaluatorDispatcher.java | 9 ++++
.../resources/OSGI-INF/blueprint/blueprint.xml | 15 ++++++
4 files changed, 52 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1c11ecc1/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index f52dc5a..2a074c8 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -430,19 +430,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
bundleContext.addBundleListener(this);
- try {
- for (ServiceReference<ConditionEvaluator> reference : bundleContext.getServiceReferences(ConditionEvaluator.class, null)) {
- ConditionEvaluator service = bundleContext.getService(reference);
- conditionEvaluatorDispatcher.addEvaluator(reference.getProperty("conditionEvaluatorId").toString(), reference.getBundle().getBundleId(), service);
- }
- for (ServiceReference<ConditionESQueryBuilder> reference : bundleContext.getServiceReferences(ConditionESQueryBuilder.class, null)) {
- ConditionESQueryBuilder service = bundleContext.getService(reference);
- conditionESQueryBuilderDispatcher.addQueryBuilder(reference.getProperty("queryBuilderId").toString(), reference.getBundle().getBundleId(), service);
- }
- } catch (Exception e) {
- logger.error("Cannot get services", e);
- }
-
timer = new Timer();
timer.scheduleAtFixedRate(new TimerTask() {
@@ -594,30 +581,30 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
bundleContext.removeBundleListener(this);
}
+ public void bindConditionEvaluator(ServiceReference<ConditionEvaluator> conditionEvaluatorServiceReference) {
+ ConditionEvaluator conditionEvaluator = bundleContext.getService(conditionEvaluatorServiceReference);
+ conditionEvaluatorDispatcher.addEvaluator(conditionEvaluatorServiceReference.getProperty("conditionEvaluatorId").toString(), conditionEvaluatorServiceReference.getBundle().getBundleId(), conditionEvaluator);
+ }
+
+ public void unbindConditionEvaluator(ServiceReference<ConditionEvaluator> conditionEvaluatorServiceReference) {
+ conditionEvaluatorDispatcher.removeEvaluator(conditionEvaluatorServiceReference.getProperty("conditionEvaluatorId").toString(), conditionEvaluatorServiceReference.getBundle().getBundleId());
+ }
+
+ public void bindConditionESQueryBuilder(ServiceReference<ConditionESQueryBuilder> conditionESQueryBuilderServiceReference) {
+ ConditionESQueryBuilder conditionESQueryBuilder = bundleContext.getService(conditionESQueryBuilderServiceReference);
+ conditionESQueryBuilderDispatcher.addQueryBuilder(conditionESQueryBuilderServiceReference.getProperty("queryBuilderId").toString(), conditionESQueryBuilderServiceReference.getBundle().getBundleId(), conditionESQueryBuilder);
+ }
+
+ public void unbindConditionESQueryBuilder(ServiceReference<ConditionESQueryBuilder> conditionESQueryBuilderServiceReference) {
+ conditionESQueryBuilderDispatcher.removeQueryBuilder(conditionESQueryBuilderServiceReference.getProperty("queryBuilderId").toString(), conditionESQueryBuilderServiceReference.getBundle().getBundleId());
+ }
+
@Override
public void bundleChanged(BundleEvent event) {
switch (event.getType()) {
- case BundleEvent.STARTED:
- // @todo replace this with a proper service tracker/listener
- if (event.getBundle() != null && event.getBundle().getRegisteredServices() != null) {
- for (ServiceReference<?> reference : event.getBundle().getRegisteredServices()) {
- Object service = bundleContext.getService(reference);
- if (service instanceof ConditionEvaluator) {
- conditionEvaluatorDispatcher.addEvaluator(reference.getProperty("conditionEvaluatorId").toString(), event.getBundle().getBundleId(), (ConditionEvaluator) service);
- }
- if (service instanceof ConditionESQueryBuilder) {
- conditionESQueryBuilderDispatcher.addQueryBuilder(reference.getProperty("queryBuilderId").toString(), event.getBundle().getBundleId(), (ConditionESQueryBuilder) service);
- }
- }
- }
- break;
case BundleEvent.STARTING:
loadPredefinedMappings(event.getBundle().getBundleContext(), true);
break;
- case BundleEvent.STOPPING:
- conditionEvaluatorDispatcher.removeEvaluators(event.getBundle().getBundleId());
- conditionESQueryBuilderDispatcher.removeQueryBuilders(event.getBundle().getBundleId());
- break;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1c11ecc1/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java
index 3ab0d5e..fbea30e 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java
@@ -52,6 +52,16 @@ public class ConditionESQueryBuilderDispatcher {
queryBuildersByBundle.get(bundleId).add(name);
}
+ public void removeQueryBuilder(String name, long bundleId) {
+ queryBuilders.remove(name);
+ List<String> bundleEvaluators = queryBuildersByBundle.get(bundleId);
+ if (bundleEvaluators != null && bundleEvaluators.size() > 0) {
+ bundleEvaluators.remove(name);
+ queryBuildersByBundle.put(bundleId, bundleEvaluators);
+ }
+ }
+
+
public void removeQueryBuilders(long bundleId) {
if (queryBuildersByBundle.containsKey(bundleId)) {
for (String s : queryBuildersByBundle.get(bundleId)) {
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1c11ecc1/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionEvaluatorDispatcher.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionEvaluatorDispatcher.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionEvaluatorDispatcher.java
index 0341c46..5573eed 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionEvaluatorDispatcher.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionEvaluatorDispatcher.java
@@ -51,6 +51,15 @@ public class ConditionEvaluatorDispatcher {
evaluatorsByBundle.get(bundleId).add(name);
}
+ public void removeEvaluator(String name, long bundleId) {
+ evaluators.remove(name);
+ List<String> bundleEvaluators = evaluatorsByBundle.get(bundleId);
+ if (bundleEvaluators != null && bundleEvaluators.size() > 0) {
+ bundleEvaluators.remove(name);
+ evaluatorsByBundle.put(bundleId, bundleEvaluators);
+ }
+ }
+
public void removeEvaluators(long bundleId) {
if (evaluatorsByBundle.containsKey(bundleId)) {
for (String s : evaluatorsByBundle.get(bundleId)) {
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1c11ecc1/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index 23963ac..8b514d7 100644
--- a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -131,4 +131,19 @@
<property name="karafJMXPort" value="${es.cluster.jmxPort}" />
</bean>
+ <!-- We use a listener here because using the list directly for listening to proxies coming from the same bundle didn't seem to work -->
+ <reference-list id="conditionEvaluators"
+ interface="org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator"
+ availability="optional">
+ <reference-listener
+ bind-method="bindConditionEvaluator" unbind-method="unbindConditionEvaluator" ref="elasticSearchPersistenceServiceImpl"/>
+ </reference-list>
+
+ <reference-list id="conditionESQueryBuilders"
+ interface="org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder"
+ availability="optional">
+ <reference-listener
+ bind-method="bindConditionESQueryBuilder" unbind-method="unbindConditionESQueryBuilder" ref="elasticSearchPersistenceServiceImpl"/>
+ </reference-list>
+
</blueprint>
[14/50] [abbrv] incubator-unomi git commit: Merge remote-tracking
branch 'origin/feature-UNOMI-28-ES2X' into feature-UNOMI-70-ES5X
Posted by sh...@apache.org.
Merge remote-tracking branch 'origin/feature-UNOMI-28-ES2X' into feature-UNOMI-70-ES5X
# Conflicts:
# api/pom.xml
# extensions/geonames/pom.xml
# extensions/geonames/rest/pom.xml
# extensions/geonames/services/pom.xml
# extensions/lists-extension/pom.xml
# extensions/lists-extension/rest/pom.xml
# extensions/lists-extension/services/pom.xml
# extensions/pom.xml
# extensions/privacy-extension/pom.xml
# extensions/privacy-extension/rest/pom.xml
# extensions/privacy-extension/services/pom.xml
# itests/pom.xml
# kar/pom.xml
# package/pom.xml
# performance-tests/pom.xml
# persistence-elasticsearch/core/pom.xml
# persistence-elasticsearch/plugins/pom.xml
# persistence-elasticsearch/plugins/security/pom.xml
# persistence-elasticsearch/pom.xml
# persistence-spi/pom.xml
# plugins/baseplugin/pom.xml
# plugins/hover-event/pom.xml
# plugins/mail/pom.xml
# plugins/optimization-test/pom.xml
# plugins/past-event/pom.xml
# plugins/pom.xml
# plugins/request/pom.xml
# plugins/tracked-event/pom.xml
# pom.xml
# rest/pom.xml
# samples/pom.xml
# samples/tweet-button-plugin/pom.xml
# services/pom.xml
# wab/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/90eb893c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/90eb893c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/90eb893c
Branch: refs/heads/master
Commit: 90eb893c747259cb1d44a32b4c2c4e2258bc6777
Parents: ff87e2f 2263da9
Author: Serge Huber <sh...@apache.org>
Authored: Mon Dec 5 10:00:32 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Mon Dec 5 10:00:32 2016 +0100
----------------------------------------------------------------------
.../META-INF/cxs/mappings/geonameEntry.json | 2 +-
itests/pom.xml | 5 +-
.../java/org/apache/unomi/itests/AllTests.java | 11 +-
.../apache/unomi/itests/ProfileServiceTest.java | 53 +++
kar/src/main/feature/feature.xml | 12 +-
package/pom.xml | 53 ++-
persistence-elasticsearch/core/pom.xml | 202 +++------
.../ElasticSearchPersistenceServiceImpl.java | 423 ++++++++++++++++---
.../conditions/ConditionContextHelper.java | 4 +-
.../conditions/ConditionESQueryBuilder.java | 4 +-
.../ConditionESQueryBuilderDispatcher.java | 16 +-
.../META-INF/cxs/mappings/campaign.json | 4 +-
.../META-INF/cxs/mappings/campaignevent.json | 2 +-
.../resources/META-INF/cxs/mappings/event.json | 2 +-
.../META-INF/cxs/mappings/profile.json | 2 +-
.../META-INF/cxs/mappings/session.json | 4 +-
.../resources/OSGI-INF/blueprint/blueprint.xml | 13 +
.../core/src/main/resources/elasticsearch.yml | 381 +++--------------
...g.apache.unomi.persistence.elasticsearch.cfg | 10 +
.../plugin/security/SecurityPlugin.java | 14 +-
.../plugin/security/SecurityPluginModule.java | 3 +-
.../plugin/security/SecurityPluginService.java | 4 +-
.../BooleanConditionESQueryBuilder.java | 20 +-
...onByPointSessionConditionESQueryBuilder.java | 10 +-
.../MatchAllConditionESQueryBuilder.java | 8 +-
.../conditions/NotConditionESQueryBuilder.java | 8 +-
.../PastEventConditionESQueryBuilder.java | 8 +-
.../PropertyConditionESQueryBuilder.java | 63 +--
.../conditions/PropertyConditionEvaluator.java | 3 +-
...rceEventPropertyConditionESQueryBuilder.java | 27 +-
.../HoverEventConditionESQueryBuilder.java | 23 +-
pom.xml | 8 +-
.../services/services/SegmentServiceImpl.java | 22 +-
33 files changed, 727 insertions(+), 697 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/90eb893c/itests/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/90eb893c/package/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/90eb893c/persistence-elasticsearch/core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/90eb893c/pom.xml
----------------------------------------------------------------------
[23/50] [abbrv] incubator-unomi git commit: Upgrade to ElasticSearch
5.1.1
Posted by sh...@apache.org.
Upgrade to ElasticSearch 5.1.1
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/1cd827a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/1cd827a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/1cd827a1
Branch: refs/heads/master
Commit: 1cd827a1252c07e9083aee9ec81104ab66b2881f
Parents: 923e491
Author: Serge Huber <sh...@apache.org>
Authored: Thu Dec 15 21:55:52 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Thu Dec 15 21:55:52 2016 +0100
----------------------------------------------------------------------
.../elasticsearch/ElasticSearchPersistenceServiceImpl.java | 4 ++--
.../baseplugin/conditions/PropertyConditionEvaluator.java | 7 +++----
pom.xml | 2 +-
3 files changed, 6 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1cd827a1/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 0651c3f..020d1d4 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -63,7 +63,7 @@ import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.*;
@@ -827,7 +827,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
String index = indexNames.containsKey(itemType) ? indexNames.get(itemType) :
(itemsMonthlyIndexed.contains(itemType) && dateHint != null ? getMonthlyIndex(dateHint) : indexName);
- Script actualScript = new Script(script, ScriptService.ScriptType.INLINE, null, scriptParams);
+ Script actualScript = new Script(ScriptType.INLINE, "groovy", script, scriptParams);
if (bulkProcessor == null) {
client.prepareUpdate(index, itemType, itemId).setScript(actualScript)
.execute()
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1cd827a1/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
index 67404a8..68e58a6 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
@@ -40,7 +40,7 @@ import org.slf4j.LoggerFactory;
import java.text.SimpleDateFormat;
import java.util.*;
-import java.util.concurrent.Callable;
+import java.util.function.LongSupplier;
import java.util.regex.Pattern;
/**
@@ -282,9 +282,8 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
} else {
DateMathParser parser = new DateMathParser(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER);
try {
- return new Date(parser.parse(value.toString(), new Callable<Long>() {
- @Override
- public Long call() throws Exception {
+ return new Date(parser.parse(value.toString(), new LongSupplier() {
+ public long getAsLong() {
return System.currentTimeMillis();
}
}));
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1cd827a1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 040e382..1aba4e5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -69,7 +69,7 @@
<version.karaf>3.0.8</version.karaf>
<version.karaf.cellar>3.0.3</version.karaf.cellar>
<version.pax.exam>4.9.1</version.pax.exam>
- <elasticsearch.version>5.0.1</elasticsearch.version>
+ <elasticsearch.version>5.1.1</elasticsearch.version>
<maven.compiler.source>1.7</maven.compiler.source>
<maven.compiler.target>1.7</maven.compiler.target>
[45/50] [abbrv] incubator-unomi git commit: DMF-1133 Error when
displaying an empty list in MF - Fixed by not generating sorts or
aggregations if there is no data for a field.
Posted by sh...@apache.org.
DMF-1133 Error when displaying an empty list in MF
- Fixed by not generating sorts or aggregations if there is no data for a field.
Signed-off-by: Serge Huber <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/b8123499
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/b8123499
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/b8123499
Branch: refs/heads/master
Commit: b81234998fb227b2727d51d77b42f13f674b29bc
Parents: f941850
Author: Serge Huber <sh...@apache.org>
Authored: Thu Jan 5 10:12:43 2017 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Thu Jan 5 10:12:43 2017 +0100
----------------------------------------------------------------------
.../ElasticSearchPersistenceServiceImpl.java | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/b8123499/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 0958bcd..f98854b 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -1300,11 +1300,16 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
} else {
String name = getPropertyNameWithData(StringUtils.substringBeforeLast(sortByElement,":"), itemType);
- if (sortByElement.endsWith(":desc")) {
- requestBuilder = requestBuilder.addSort(name, SortOrder.DESC);
+ if (name != null) {
+ if (sortByElement.endsWith(":desc")) {
+ requestBuilder = requestBuilder.addSort(name, SortOrder.DESC);
+ } else {
+ requestBuilder = requestBuilder.addSort(name, SortOrder.ASC);
+ }
} else {
- requestBuilder = requestBuilder.addSort(name, SortOrder.ASC);
+ // in the case of no data existing for the property, we will not add the sorting to the request.
}
+
}
}
}
@@ -1456,7 +1461,11 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
} else {
fieldName = getPropertyNameWithData(fieldName, itemType);
//default
- bucketsAggregation = AggregationBuilders.terms("buckets").field(fieldName).size(5000);
+ if (fieldName != null) {
+ bucketsAggregation = AggregationBuilders.terms("buckets").field(fieldName).size(5000);
+ } else {
+ // field name could be null if no existing data exists
+ }
}
if (bucketsAggregation != null) {
final MissingAggregationBuilder missingBucketsAggregation = AggregationBuilders.missing("missing").field(fieldName);
[07/50] [abbrv] incubator-unomi git commit: UNOMI-63 Use
ElasticSearch BulkProcessing to perform segment updates - Use configured
address by default if none is specified in the elasticsearch.yml file.
Posted by sh...@apache.org.
UNOMI-63 Use ElasticSearch BulkProcessing to perform segment updates
- Use configured address by default if none is specified in the elasticsearch.yml file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/62d11ded
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/62d11ded
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/62d11ded
Branch: refs/heads/master
Commit: 62d11ded05668452731af5ba4ae23e8cd6bb98c2
Parents: fcea705
Author: Serge Huber <sh...@apache.org>
Authored: Thu Nov 17 14:29:12 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Thu Nov 17 14:29:12 2016 +0100
----------------------------------------------------------------------
.../elasticsearch/ElasticSearchPersistenceServiceImpl.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/62d11ded/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 2915c32..b954b75 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -143,6 +143,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
public static final String BULK_PROCESSOR_BULK_SIZE = "bulkProcessor.bulkSize";
public static final String BULK_PROCESSOR_FLUSH_INTERVAL = "bulkProcessor.flushInterval";
public static final String BULK_PROCESSOR_BACKOFF_POLICY = "bulkProcessor.backoffPolicy";
+ public static final String ELASTICSEARCH_NETWORK_HOST = "network.host";
private Node node;
private Client client;
@@ -349,6 +350,11 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
.put(NODE_CONTEXTSERVER_SECURE_PORT, securePort)
.put(INDEX_MAX_RESULT_WINDOW, "2147483647");
+ if (settingsBuilder.get(ELASTICSEARCH_NETWORK_HOST) == null) {
+ logger.info("Setting ElasticSearch network host address to {}", address);
+ settingsBuilder.put(ELASTICSEARCH_NETWORK_HOST, address);
+ }
+
node = nodeBuilder().settings(settingsBuilder).node();
client = node.client();
logger.info("Waiting for ElasticSearch to start...");
[42/50] [abbrv] incubator-unomi git commit: Fix port in cluster
configuration
Posted by sh...@apache.org.
Fix port in cluster configuration
Signed-off-by: Serge Huber <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/adb5fae9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/adb5fae9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/adb5fae9
Branch: refs/heads/master
Commit: adb5fae90660606ed478301b07356b96182540e9
Parents: 2105da7
Author: Serge Huber <sh...@apache.org>
Authored: Thu Dec 22 20:42:00 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Thu Dec 22 20:42:00 2016 +0100
----------------------------------------------------------------------
.../elasticsearch/ElasticSearchPersistenceServiceImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/adb5fae9/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 8fc7a7e..b3b569c 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -340,7 +340,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
String[] secureEndpointsArray = secureEndpointsPropValue.split(",");
Set<String> secureEndpoints = new TreeSet<String>(Arrays.asList(secureEndpointsArray));
publicEndpoints.add(thisKarafNode.getId() + "=" + address + ":" + port);
- secureEndpoints.add(thisKarafNode.getId() + "=" + secureAddress + ":" + port);
+ secureEndpoints.add(thisKarafNode.getId() + "=" + secureAddress + ":" + securePort);
karafCellarClusterNodeConfiguration.setProperty(KARAF_CLUSTER_CONFIGURATION_PUBLIC_ENDPOINTS, StringUtils.join(publicEndpoints, ","));
karafCellarClusterNodeConfiguration.setProperty(KARAF_CLUSTER_CONFIGURATION_SECURE_ENDPOINTS, StringUtils.join(secureEndpoints, ","));
configurations.put(KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION, karafCellarClusterNodeConfiguration);
[41/50] [abbrv] incubator-unomi git commit: Fixed most issues by
making sure we use reference listeners everywhere for proper registration of
actions and conditions
Posted by sh...@apache.org.
Fixed most issues by making sure we use reference listeners everywhere for proper registration of actions and conditions
Signed-off-by: Serge Huber <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/2105da7e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/2105da7e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/2105da7e
Branch: refs/heads/master
Commit: 2105da7eceeecb377063dca34bdbb545042691fa
Parents: 1c11ecc
Author: Serge Huber <sh...@apache.org>
Authored: Thu Dec 22 17:25:29 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Thu Dec 22 17:25:29 2016 +0100
----------------------------------------------------------------------
.../ElasticSearchPersistenceServiceImpl.java | 14 +++++---
.../ConditionESQueryBuilderDispatcher.java | 31 ++--------------
.../ConditionEvaluatorDispatcher.java | 31 ++--------------
.../resources/OSGI-INF/blueprint/blueprint.xml | 2 --
.../actions/ActionExecutorDispatcher.java | 24 ++-----------
.../services/services/RulesServiceImpl.java | 37 +++++++++-----------
.../resources/OSGI-INF/blueprint/blueprint.xml | 8 ++++-
7 files changed, 40 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/2105da7e/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 2a074c8..8fc7a7e 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -583,20 +583,26 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
public void bindConditionEvaluator(ServiceReference<ConditionEvaluator> conditionEvaluatorServiceReference) {
ConditionEvaluator conditionEvaluator = bundleContext.getService(conditionEvaluatorServiceReference);
- conditionEvaluatorDispatcher.addEvaluator(conditionEvaluatorServiceReference.getProperty("conditionEvaluatorId").toString(), conditionEvaluatorServiceReference.getBundle().getBundleId(), conditionEvaluator);
+ conditionEvaluatorDispatcher.addEvaluator(conditionEvaluatorServiceReference.getProperty("conditionEvaluatorId").toString(), conditionEvaluator);
}
public void unbindConditionEvaluator(ServiceReference<ConditionEvaluator> conditionEvaluatorServiceReference) {
- conditionEvaluatorDispatcher.removeEvaluator(conditionEvaluatorServiceReference.getProperty("conditionEvaluatorId").toString(), conditionEvaluatorServiceReference.getBundle().getBundleId());
+ if (conditionEvaluatorServiceReference == null) {
+ return;
+ }
+ conditionEvaluatorDispatcher.removeEvaluator(conditionEvaluatorServiceReference.getProperty("conditionEvaluatorId").toString());
}
public void bindConditionESQueryBuilder(ServiceReference<ConditionESQueryBuilder> conditionESQueryBuilderServiceReference) {
ConditionESQueryBuilder conditionESQueryBuilder = bundleContext.getService(conditionESQueryBuilderServiceReference);
- conditionESQueryBuilderDispatcher.addQueryBuilder(conditionESQueryBuilderServiceReference.getProperty("queryBuilderId").toString(), conditionESQueryBuilderServiceReference.getBundle().getBundleId(), conditionESQueryBuilder);
+ conditionESQueryBuilderDispatcher.addQueryBuilder(conditionESQueryBuilderServiceReference.getProperty("queryBuilderId").toString(), conditionESQueryBuilder);
}
public void unbindConditionESQueryBuilder(ServiceReference<ConditionESQueryBuilder> conditionESQueryBuilderServiceReference) {
- conditionESQueryBuilderDispatcher.removeQueryBuilder(conditionESQueryBuilderServiceReference.getProperty("queryBuilderId").toString(), conditionESQueryBuilderServiceReference.getBundle().getBundleId());
+ if (conditionESQueryBuilderServiceReference == null) {
+ return;
+ }
+ conditionESQueryBuilderDispatcher.removeQueryBuilder(conditionESQueryBuilderServiceReference.getProperty("queryBuilderId").toString());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/2105da7e/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java
index fbea30e..b3099bc 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java
@@ -20,57 +20,30 @@ package org.apache.unomi.persistence.elasticsearch.conditions;
import org.apache.unomi.api.conditions.Condition;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
-import org.osgi.framework.BundleContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class ConditionESQueryBuilderDispatcher {
private static final Logger logger = LoggerFactory.getLogger(ConditionESQueryBuilderDispatcher.class.getName());
- private BundleContext bundleContext;
private Map<String, ConditionESQueryBuilder> queryBuilders = new ConcurrentHashMap<>();
- private Map<Long, List<String>> queryBuildersByBundle = new ConcurrentHashMap<>();
public ConditionESQueryBuilderDispatcher() {
}
- public void setBundleContext(BundleContext bundleContext) {
- this.bundleContext = bundleContext;
- }
-
- public void addQueryBuilder(String name, long bundleId, ConditionESQueryBuilder evaluator) {
+ public void addQueryBuilder(String name, ConditionESQueryBuilder evaluator) {
queryBuilders.put(name, evaluator);
- if (!queryBuildersByBundle.containsKey(bundleId)) {
- queryBuildersByBundle.put(bundleId, new ArrayList<String>());
- }
- queryBuildersByBundle.get(bundleId).add(name);
}
- public void removeQueryBuilder(String name, long bundleId) {
+ public void removeQueryBuilder(String name) {
queryBuilders.remove(name);
- List<String> bundleEvaluators = queryBuildersByBundle.get(bundleId);
- if (bundleEvaluators != null && bundleEvaluators.size() > 0) {
- bundleEvaluators.remove(name);
- queryBuildersByBundle.put(bundleId, bundleEvaluators);
- }
}
- public void removeQueryBuilders(long bundleId) {
- if (queryBuildersByBundle.containsKey(bundleId)) {
- for (String s : queryBuildersByBundle.get(bundleId)) {
- queryBuilders.remove(s);
- }
- queryBuildersByBundle.remove(bundleId);
- }
- }
-
public String getQuery(Condition condition) {
return "{\"query\": " + getQueryBuilder(condition).toString() + "}";
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/2105da7e/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionEvaluatorDispatcher.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionEvaluatorDispatcher.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionEvaluatorDispatcher.java
index 5573eed..dff7ecb 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionEvaluatorDispatcher.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionEvaluatorDispatcher.java
@@ -19,13 +19,10 @@ package org.apache.unomi.persistence.elasticsearch.conditions;
import org.apache.unomi.api.Item;
import org.apache.unomi.api.conditions.Condition;
-import org.osgi.framework.BundleContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -35,38 +32,14 @@ import java.util.concurrent.ConcurrentHashMap;
public class ConditionEvaluatorDispatcher {
private static final Logger logger = LoggerFactory.getLogger(ConditionEvaluatorDispatcher.class.getName());
- private BundleContext bundleContext;
private Map<String, ConditionEvaluator> evaluators = new ConcurrentHashMap<>();
- private Map<Long, List<String>> evaluatorsByBundle = new ConcurrentHashMap<>();
- public void setBundleContext(BundleContext bundleContext) {
- this.bundleContext = bundleContext;
- }
-
- public void addEvaluator(String name, long bundleId, ConditionEvaluator evaluator) {
+ public void addEvaluator(String name, ConditionEvaluator evaluator) {
evaluators.put(name, evaluator);
- if (!evaluatorsByBundle.containsKey(bundleId)) {
- evaluatorsByBundle.put(bundleId, new ArrayList<String>());
- }
- evaluatorsByBundle.get(bundleId).add(name);
}
- public void removeEvaluator(String name, long bundleId) {
+ public void removeEvaluator(String name) {
evaluators.remove(name);
- List<String> bundleEvaluators = evaluatorsByBundle.get(bundleId);
- if (bundleEvaluators != null && bundleEvaluators.size() > 0) {
- bundleEvaluators.remove(name);
- evaluatorsByBundle.put(bundleId, bundleEvaluators);
- }
- }
-
- public void removeEvaluators(long bundleId) {
- if (evaluatorsByBundle.containsKey(bundleId)) {
- for (String s : evaluatorsByBundle.get(bundleId)) {
- evaluators.remove(s);
- }
- evaluatorsByBundle.remove(bundleId);
- }
}
public boolean eval(Condition condition, Item item) {
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/2105da7e/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index 8b514d7..fea1879 100644
--- a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -75,12 +75,10 @@
<bean id="conditionESQueryBuilderDispatcher"
class="org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher">
- <property name="bundleContext" ref="blueprintBundleContext"/>
</bean>
<bean id="conditionEvaluatorDispatcherImpl"
class="org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorDispatcher">
- <property name="bundleContext" ref="blueprintBundleContext"/>
</bean>
<bean id="elasticSearchPersistenceServiceImpl"
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/2105da7e/services/src/main/java/org/apache/unomi/services/actions/ActionExecutorDispatcher.java
----------------------------------------------------------------------
diff --git a/services/src/main/java/org/apache/unomi/services/actions/ActionExecutorDispatcher.java b/services/src/main/java/org/apache/unomi/services/actions/ActionExecutorDispatcher.java
index bcd49e7..62e2463 100644
--- a/services/src/main/java/org/apache/unomi/services/actions/ActionExecutorDispatcher.java
+++ b/services/src/main/java/org/apache/unomi/services/actions/ActionExecutorDispatcher.java
@@ -26,15 +26,12 @@ import org.apache.unomi.api.services.EventService;
import org.mvel2.MVEL;
import org.mvel2.ParserConfiguration;
import org.mvel2.ParserContext;
-import org.osgi.framework.BundleContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Serializable;
import java.lang.reflect.InvocationTargetException;
-import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -43,9 +40,7 @@ public class ActionExecutorDispatcher {
private static final String VALUE_NAME_SEPARATOR = "::";
private final Map<String, Serializable> mvelExpressions = new ConcurrentHashMap<>();
private final Map<String, ValueExtractor> valueExtractors = new HashMap<>(11);
- private BundleContext bundleContext;
private Map<String, ActionExecutor> executors = new ConcurrentHashMap<>();
- private Map<Long, List<String>> executorsByBundle = new ConcurrentHashMap<>();
public ActionExecutorDispatcher() {
valueExtractors.put("profileProperty", new ValueExtractor() {
@@ -101,21 +96,12 @@ public class ActionExecutorDispatcher {
});
}
- public void addExecutor(String name, long bundleId, ActionExecutor evaluator) {
+ public void addExecutor(String name, ActionExecutor evaluator) {
executors.put(name, evaluator);
- if (!executorsByBundle.containsKey(bundleId)) {
- executorsByBundle.put(bundleId, new ArrayList<String>());
- }
- executorsByBundle.get(bundleId).add(name);
}
- public void removeExecutors(long bundleId) {
- if (executorsByBundle.containsKey(bundleId)) {
- for (String s : executorsByBundle.get(bundleId)) {
- executors.remove(s);
- }
- executorsByBundle.remove(bundleId);
- }
+ public void removeExecutor(String name) {
+ executors.remove(name);
}
public Action getContextualAction(Action action, Event event) {
@@ -177,10 +163,6 @@ public class ActionExecutorDispatcher {
return false;
}
- public void setBundleContext(BundleContext bundleContext) {
- this.bundleContext = bundleContext;
- }
-
public int execute(Action action, Event event) {
String actionKey = action.getActionType().getActionExecutor();
if (actionKey == null) {
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/2105da7e/services/src/main/java/org/apache/unomi/services/services/RulesServiceImpl.java
----------------------------------------------------------------------
diff --git a/services/src/main/java/org/apache/unomi/services/services/RulesServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/RulesServiceImpl.java
index b1d4d51..e4deeb1 100644
--- a/services/src/main/java/org/apache/unomi/services/services/RulesServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/services/RulesServiceImpl.java
@@ -17,12 +17,13 @@
package org.apache.unomi.services.services;
-import org.apache.unomi.api.*;
+import org.apache.unomi.api.Event;
+import org.apache.unomi.api.Item;
+import org.apache.unomi.api.Metadata;
+import org.apache.unomi.api.PartialList;
import org.apache.unomi.api.actions.Action;
import org.apache.unomi.api.actions.ActionExecutor;
-import org.apache.unomi.api.actions.ActionType;
import org.apache.unomi.api.conditions.Condition;
-import org.apache.unomi.api.conditions.ConditionType;
import org.apache.unomi.api.query.Query;
import org.apache.unomi.api.rules.Rule;
import org.apache.unomi.api.services.DefinitionsService;
@@ -77,6 +78,18 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn
this.actionExecutorDispatcher = actionExecutorDispatcher;
}
+ public void bindExecutor(ServiceReference<ActionExecutor> actionExecutorServiceReference) {
+ ActionExecutor actionExecutor = bundleContext.getService(actionExecutorServiceReference);
+ actionExecutorDispatcher.addExecutor(actionExecutorServiceReference.getProperty("actionExecutorId").toString(), actionExecutor);
+ }
+
+ public void unbindExecutor(ServiceReference<ActionExecutor> actionExecutorServiceReference) {
+ if (actionExecutorServiceReference == null) {
+ return;
+ }
+ actionExecutorDispatcher.removeExecutor(actionExecutorServiceReference.getProperty("actionExecutorId").toString());
+ }
+
public void postConstruct() {
logger.debug("postConstruct {" + bundleContext.getBundle() + "}");
@@ -86,14 +99,6 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn
loadPredefinedRules(bundle.getBundleContext());
}
}
- try {
- for (ServiceReference<ActionExecutor> reference : bundleContext.getServiceReferences(ActionExecutor.class, null)) {
- ActionExecutor service = bundleContext.getService(reference);
- actionExecutorDispatcher.addExecutor(reference.getProperty("actionExecutorId").toString(), reference.getBundle().getBundleId(), service);
- }
- } catch (Exception e) {
- logger.error("Cannot get services",e);
- }
bundleContext.addBundleListener(this);
@@ -117,22 +122,12 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn
return;
}
loadPredefinedRules(bundleContext);
-
- if (bundleContext.getBundle().getRegisteredServices() != null) {
- for (ServiceReference<?> reference : bundleContext.getBundle().getRegisteredServices()) {
- Object service = bundleContext.getService(reference);
- if (service instanceof ActionExecutor) {
- actionExecutorDispatcher.addExecutor(reference.getProperty("actionExecutorId").toString(), bundleContext.getBundle().getBundleId(), (ActionExecutor) service);
- }
- }
- }
}
private void processBundleStop(BundleContext bundleContext) {
if (bundleContext == null) {
return;
}
- actionExecutorDispatcher.removeExecutors(bundleContext.getBundle().getBundleId());
}
private void loadPredefinedRules(BundleContext bundleContext) {
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/2105da7e/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
----------------------------------------------------------------------
diff --git a/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index 3176639..8c0d371 100644
--- a/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ b/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -84,7 +84,6 @@
<bean id="actionExecutorDispatcherImpl"
class="org.apache.unomi.services.actions.ActionExecutorDispatcher">
- <property name="bundleContext" ref="blueprintBundleContext"/>
</bean>
<bean id="rulesServiceImpl" class="org.apache.unomi.services.services.RulesServiceImpl"
@@ -138,6 +137,13 @@
bind-method="bind" unbind-method="unbind" ref="eventServiceImpl"/>
</reference-list>
+ <reference-list id="actionExecutors"
+ interface="org.apache.unomi.api.actions.ActionExecutor"
+ availability="optional">
+ <reference-listener
+ bind-method="bindExecutor" unbind-method="unbindExecutor" ref="rulesServiceImpl"/>
+ </reference-list>
+
<!-- Property merge strategy executors -->
<service auto-export="interfaces">
[46/50] [abbrv] incubator-unomi git commit: - DMF-1133 Error when
displaying an empty list in MF : fixed issue for real this time :) - Some
configuration file cleanup - Changed inactive user purging from 30 days to
180 days
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/services/src/main/java/org/apache/unomi/services/services/ClusterServiceImpl.java
----------------------------------------------------------------------
diff --git a/services/src/main/java/org/apache/unomi/services/services/ClusterServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/ClusterServiceImpl.java
new file mode 100644
index 0000000..8fb67d3
--- /dev/null
+++ b/services/src/main/java/org/apache/unomi/services/services/ClusterServiceImpl.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.unomi.services.services;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.karaf.cellar.config.ClusterConfigurationEvent;
+import org.apache.karaf.cellar.config.Constants;
+import org.apache.karaf.cellar.core.*;
+import org.apache.karaf.cellar.core.control.SwitchStatus;
+import org.apache.karaf.cellar.core.event.EventProducer;
+import org.apache.karaf.cellar.core.event.EventType;
+import org.apache.unomi.api.ClusterNode;
+import org.apache.unomi.api.services.ClusterService;
+import org.apache.unomi.persistence.spi.PersistenceService;
+import org.osgi.service.cm.ConfigurationAdmin;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.management.*;
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXConnectorFactory;
+import javax.management.remote.JMXServiceURL;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
+import java.lang.management.RuntimeMXBean;
+import java.net.MalformedURLException;
+import java.util.*;
+
+/**
+ * Implementation of the persistence service interface
+ */
+public class ClusterServiceImpl implements ClusterService {
+
+ private static final Logger logger = LoggerFactory.getLogger(ClusterServiceImpl.class.getName());
+
+ public static final String CONTEXTSERVER_ADDRESS = "contextserver.address";
+ public static final String CONTEXTSERVER_PORT = "contextserver.port";
+ public static final String CONTEXTSERVER_SECURE_ADDRESS = "contextserver.secureAddress";
+ public static final String CONTEXTSERVER_SECURE_PORT = "contextserver.securePort";
+ public static final String KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION = "org.apache.unomi.nodes";
+ public static final String KARAF_CLUSTER_CONFIGURATION_PUBLIC_ENDPOINTS = "publicEndpoints";
+ public static final String KARAF_CLUSTER_CONFIGURATION_SECURE_ENDPOINTS = "secureEndpoints";
+
+ private ClusterManager karafCellarClusterManager;
+ private EventProducer karafCellarEventProducer;
+ private GroupManager karafCellarGroupManager;
+ private String karafCellarGroupName = Configurations.DEFAULT_GROUP_NAME;
+ private ConfigurationAdmin osgiConfigurationAdmin;
+ private String karafJMXUsername = "karaf";
+ private String karafJMXPassword = "karaf";
+ private int karafJMXPort = 1099;
+ private String address;
+ private String port;
+ private String secureAddress;
+ private String securePort;
+
+ PersistenceService persistenceService;
+
+ public void setPersistenceService(PersistenceService persistenceService) {
+ this.persistenceService = persistenceService;
+ }
+
+ public void setKarafCellarClusterManager(ClusterManager karafCellarClusterManager) {
+ this.karafCellarClusterManager = karafCellarClusterManager;
+ }
+
+ public void setKarafCellarEventProducer(EventProducer karafCellarEventProducer) {
+ this.karafCellarEventProducer = karafCellarEventProducer;
+ }
+
+ public void setKarafCellarGroupManager(GroupManager karafCellarGroupManager) {
+ this.karafCellarGroupManager = karafCellarGroupManager;
+ }
+
+ public void setKarafCellarGroupName(String karafCellarGroupName) {
+ this.karafCellarGroupName = karafCellarGroupName;
+ }
+
+ public void setOsgiConfigurationAdmin(ConfigurationAdmin osgiConfigurationAdmin) {
+ this.osgiConfigurationAdmin = osgiConfigurationAdmin;
+ }
+
+ public void setKarafJMXUsername(String karafJMXUsername) {
+ this.karafJMXUsername = karafJMXUsername;
+ }
+
+ public void setKarafJMXPassword(String karafJMXPassword) {
+ this.karafJMXPassword = karafJMXPassword;
+ }
+
+ public void setKarafJMXPort(int karafJMXPort) {
+ this.karafJMXPort = karafJMXPort;
+ }
+
+ public void setAddress(String address) {
+ this.address = address;
+ }
+
+ public void setPort(String port) {
+ this.port = port;
+ }
+
+ public void setSecureAddress(String secureAddress) {
+ this.secureAddress = secureAddress;
+ }
+
+ public void setSecurePort(String securePort) {
+ this.securePort = securePort;
+ }
+
+ public void init() {
+ logger.debug("init cluster service");
+ if (karafCellarEventProducer != null && karafCellarClusterManager != null) {
+
+ address = System.getProperty(CONTEXTSERVER_ADDRESS, address);
+ port = System.getProperty(CONTEXTSERVER_PORT, port);
+ secureAddress = System.getProperty(CONTEXTSERVER_SECURE_ADDRESS, secureAddress);
+ securePort = System.getProperty(CONTEXTSERVER_SECURE_PORT, securePort);
+
+ boolean setupConfigOk = true;
+ Group group = karafCellarGroupManager.findGroupByName(karafCellarGroupName);
+ if (setupConfigOk && group == null) {
+ logger.error("Cluster group " + karafCellarGroupName + " doesn't exist");
+ setupConfigOk = false;
+ }
+
+ // check if the producer is ON
+ if (setupConfigOk && karafCellarEventProducer.getSwitch().getStatus().equals(SwitchStatus.OFF)) {
+ logger.error("Cluster event producer is OFF");
+ setupConfigOk = false;
+ }
+
+ // check if the config pid is allowed
+ if (setupConfigOk && !isClusterConfigPIDAllowed(group, Constants.CATEGORY, KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION, EventType.OUTBOUND)) {
+ logger.error("Configuration PID " + KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION + " is blocked outbound for cluster group " + karafCellarGroupName);
+ setupConfigOk = false;
+ }
+
+ if (setupConfigOk) {
+ Map<String, Properties> configurations = karafCellarClusterManager.getMap(Constants.CONFIGURATION_MAP + Configurations.SEPARATOR + karafCellarGroupName);
+ org.apache.karaf.cellar.core.Node thisKarafNode = karafCellarClusterManager.getNode();
+ Properties karafCellarClusterNodeConfiguration = configurations.get(KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION);
+ if (karafCellarClusterNodeConfiguration == null) {
+ karafCellarClusterNodeConfiguration = new Properties();
+ }
+ String publicEndpointsPropValue = karafCellarClusterNodeConfiguration.getProperty(KARAF_CLUSTER_CONFIGURATION_PUBLIC_ENDPOINTS, thisKarafNode.getId() + "=" + address + ":" + port);
+ String secureEndpointsPropValue = karafCellarClusterNodeConfiguration.getProperty(KARAF_CLUSTER_CONFIGURATION_SECURE_ENDPOINTS, thisKarafNode.getId() + "=" + secureAddress + ":" + securePort);
+ String[] publicEndpointsArray = publicEndpointsPropValue.split(",");
+ Set<String> publicEndpoints = new TreeSet<String>(Arrays.asList(publicEndpointsArray));
+ String[] secureEndpointsArray = secureEndpointsPropValue.split(",");
+ Set<String> secureEndpoints = new TreeSet<String>(Arrays.asList(secureEndpointsArray));
+ publicEndpoints.add(thisKarafNode.getId() + "=" + address + ":" + port);
+ secureEndpoints.add(thisKarafNode.getId() + "=" + secureAddress + ":" + securePort);
+ karafCellarClusterNodeConfiguration.setProperty(KARAF_CLUSTER_CONFIGURATION_PUBLIC_ENDPOINTS, StringUtils.join(publicEndpoints, ","));
+ karafCellarClusterNodeConfiguration.setProperty(KARAF_CLUSTER_CONFIGURATION_SECURE_ENDPOINTS, StringUtils.join(secureEndpoints, ","));
+ configurations.put(KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION, karafCellarClusterNodeConfiguration);
+ ClusterConfigurationEvent clusterConfigurationEvent = new ClusterConfigurationEvent(KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION);
+ clusterConfigurationEvent.setSourceGroup(group);
+ karafCellarEventProducer.produce(clusterConfigurationEvent);
+ }
+ }
+ }
+
+ public void destroy() {
+ }
+
+ @Override
+ public List<ClusterNode> getClusterNodes() {
+ Map<String, ClusterNode> clusterNodes = new LinkedHashMap<String, ClusterNode>();
+
+ Set<org.apache.karaf.cellar.core.Node> karafCellarNodes = karafCellarClusterManager.listNodes();
+ org.apache.karaf.cellar.core.Node thisKarafNode = karafCellarClusterManager.getNode();
+ Map<String, Properties> clusterConfigurations = karafCellarClusterManager.getMap(Constants.CONFIGURATION_MAP + Configurations.SEPARATOR + karafCellarGroupName);
+ Properties karafCellarClusterNodeConfiguration = clusterConfigurations.get(KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION);
+ Map<String, String> publicNodeEndpoints = new TreeMap<>();
+ Map<String, String> secureNodeEndpoints = new TreeMap<>();
+ if (karafCellarClusterNodeConfiguration != null) {
+ String publicEndpointsPropValue = karafCellarClusterNodeConfiguration.getProperty(KARAF_CLUSTER_CONFIGURATION_PUBLIC_ENDPOINTS, thisKarafNode.getId() + "=" + address + ":" + port);
+ String secureEndpointsPropValue = karafCellarClusterNodeConfiguration.getProperty(KARAF_CLUSTER_CONFIGURATION_SECURE_ENDPOINTS, thisKarafNode.getId() + "=" + secureAddress + ":" + securePort);
+ String[] publicEndpointsArray = publicEndpointsPropValue.split(",");
+ Set<String> publicEndpoints = new TreeSet<String>(Arrays.asList(publicEndpointsArray));
+ for (String endpoint : publicEndpoints) {
+ String[] endpointParts = endpoint.split("=");
+ publicNodeEndpoints.put(endpointParts[0], endpointParts[1]);
+ }
+ String[] secureEndpointsArray = secureEndpointsPropValue.split(",");
+ Set<String> secureEndpoints = new TreeSet<String>(Arrays.asList(secureEndpointsArray));
+ for (String endpoint : secureEndpoints) {
+ String[] endpointParts = endpoint.split("=");
+ secureNodeEndpoints.put(endpointParts[0], endpointParts[1]);
+ }
+ }
+ for (org.apache.karaf.cellar.core.Node karafCellarNode : karafCellarNodes) {
+ ClusterNode clusterNode = new ClusterNode();
+ clusterNode.setHostName(karafCellarNode.getHost());
+ String publicEndpoint = publicNodeEndpoints.get(karafCellarNode.getId());
+ if (publicEndpoint != null) {
+ String[] publicEndpointParts = publicEndpoint.split(":");
+ clusterNode.setHostAddress(publicEndpointParts[0]);
+ clusterNode.setPublicPort(Integer.parseInt(publicEndpointParts[1]));
+ }
+ String secureEndpoint = secureNodeEndpoints.get(karafCellarNode.getId());
+ if (secureEndpoint != null) {
+ String[] secureEndpointParts = secureEndpoint.split(":");
+ clusterNode.setSecureHostAddress(secureEndpointParts[0]);
+ clusterNode.setSecurePort(Integer.parseInt(secureEndpointParts[1]));
+ clusterNode.setMaster(false);
+ clusterNode.setData(false);
+ }
+ try {
+ // now let's connect to remote JMX service to retrieve information from the runtime and operating system MX beans
+ JMXServiceURL url = new JMXServiceURL("service:jmx:rmi:///jndi/rmi://"+karafCellarNode.getHost() + ":"+karafJMXPort+"/karaf-root");
+ Map<String,Object> environment=new HashMap<String,Object>();
+ if (karafJMXUsername != null && karafJMXPassword != null) {
+ environment.put(JMXConnector.CREDENTIALS,new String[]{karafJMXUsername,karafJMXPassword});
+ }
+ JMXConnector jmxc = JMXConnectorFactory.connect(url, environment);
+ MBeanServerConnection mbsc = jmxc.getMBeanServerConnection();
+ final RuntimeMXBean remoteRuntime = ManagementFactory.newPlatformMXBeanProxy(mbsc, ManagementFactory.RUNTIME_MXBEAN_NAME, RuntimeMXBean.class);
+ clusterNode.setUptime(remoteRuntime.getUptime());
+ ObjectName operatingSystemMXBeanName = new ObjectName(ManagementFactory.OPERATING_SYSTEM_MXBEAN_NAME);
+ Double processCpuLoad = null;
+ Double systemCpuLoad = null;
+ try {
+ processCpuLoad = (Double) mbsc.getAttribute(operatingSystemMXBeanName, "ProcessCpuLoad");
+ } catch (MBeanException e) {
+ e.printStackTrace();
+ } catch (AttributeNotFoundException e) {
+ e.printStackTrace();
+ }
+ try {
+ systemCpuLoad = (Double) mbsc.getAttribute(operatingSystemMXBeanName, "SystemCpuLoad");
+ } catch (MBeanException e) {
+ e.printStackTrace();
+ } catch (AttributeNotFoundException e) {
+ e.printStackTrace();
+ }
+ final OperatingSystemMXBean remoteOperatingSystemMXBean = ManagementFactory.newPlatformMXBeanProxy(mbsc, ManagementFactory.OPERATING_SYSTEM_MXBEAN_NAME, OperatingSystemMXBean.class);
+ clusterNode.setLoadAverage(new double[] { remoteOperatingSystemMXBean.getSystemLoadAverage()});
+ if (systemCpuLoad != null) {
+ clusterNode.setCpuLoad(systemCpuLoad);
+ }
+
+ } catch (MalformedURLException e) {
+ logger.error("Error connecting to remote JMX server", e);
+ } catch (IOException e) {
+ logger.error("Error retrieving remote JMX data", e);
+ } catch (MalformedObjectNameException e) {
+ logger.error("Error retrieving remote JMX data", e);
+ } catch (InstanceNotFoundException e) {
+ logger.error("Error retrieving remote JMX data", e);
+ } catch (ReflectionException e) {
+ logger.error("Error retrieving remote JMX data", e);
+ }
+ clusterNodes.put(karafCellarNode.getId(), clusterNode);
+ }
+
+ return new ArrayList<ClusterNode>(clusterNodes.values());
+ }
+
+ @Override
+ public void purge(Date date) {
+ persistenceService.purge(date);
+ }
+
+ @Override
+ public void purge(String scope) {
+ persistenceService.purge(scope);
+ }
+
+ /**
+ * Check if a configuration is allowed.
+ *
+ * @param group the cluster group.
+ * @param category the configuration category constant.
+ * @param pid the configuration PID.
+ * @param type the cluster event type.
+ * @return true if the cluster event type is allowed, false else.
+ */
+ public boolean isClusterConfigPIDAllowed(Group group, String category, String pid, EventType type) {
+ CellarSupport support = new CellarSupport();
+ support.setClusterManager(this.karafCellarClusterManager);
+ support.setGroupManager(this.karafCellarGroupManager);
+ support.setConfigurationAdmin(this.osgiConfigurationAdmin);
+ return support.isAllowed(group, category, pid, type);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
----------------------------------------------------------------------
diff --git a/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index 8c0d371..cb334ab 100644
--- a/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ b/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -26,16 +26,41 @@
update-strategy="reload" placeholder-prefix="${services.">
<cm:default-properties>
<cm:property name="profile.purge.interval" value="1"/>
- <cm:property name="profile.purge.inactiveTime" value="30"/>
+ <cm:property name="profile.purge.inactiveTime" value="180"/>
<cm:property name="profile.purge.existTime" value="-1"/>
<cm:property name="event.purge.existTime" value="12"/>
<cm:property name="segment.update.batchSize" value="1000"/>
</cm:default-properties>
</cm:property-placeholder>
+ <cm:property-placeholder persistent-id="org.apache.unomi.cluster"
+ update-strategy="reload" placeholder-prefix="${cluster.">
+ <cm:default-properties>
+ <cm:property name="group" value="default" />
+ <cm:property name="jmxUsername" value="karaf" />
+ <cm:property name="jmxPassword" value="karaf" />
+ <cm:property name="jmxPort" value="1099" />
+ </cm:default-properties>
+ </cm:property-placeholder>
+
+ <cm:property-placeholder persistent-id="org.apache.unomi.web"
+ update-strategy="reload" placeholder-prefix="${web.">
+ <cm:default-properties>
+ <cm:property name="contextserver.address" value="localhost"/>
+ <cm:property name="contextserver.port" value="8181"/>
+ <cm:property name="contextserver.secureAddress" value="localhost"/>
+ <cm:property name="contextserver.securePort" value="9443"/>
+ </cm:default-properties>
+ </cm:property-placeholder>
+
+
<reference id="persistenceService"
interface="org.apache.unomi.persistence.spi.PersistenceService"/>
<reference id="httpService" interface="org.osgi.service.http.HttpService"/>
+ <reference id="karafCellarClusterManager" interface="org.apache.karaf.cellar.core.ClusterManager" />
+ <reference id="karafCellarEventProducer" interface="org.apache.karaf.cellar.core.event.EventProducer" />
+ <reference id="karafCellarGroupManager" interface="org.apache.karaf.cellar.core.GroupManager" />
+ <reference id="osgiConfigurationAdmin" interface="org.osgi.service.cm.ConfigurationAdmin"/>
<!-- Service definitions -->
@@ -129,6 +154,24 @@
</bean>
<service id="queryService" ref="queryServiceImpl" auto-export="interfaces"/>
+ <bean id="clusterServiceImpl" class="org.apache.unomi.services.services.ClusterServiceImpl"
+ init-method="init" destroy-method="destroy">
+ <property name="address" value="${web.contextserver.address}"/>
+ <property name="port" value="${web.contextserver.port}"/>
+ <property name="secureAddress" value="${web.contextserver.secureAddress}"/>
+ <property name="securePort" value="${web.contextserver.securePort}"/>
+ <property name="persistenceService" ref="persistenceService"/>
+ <property name="karafCellarClusterManager" ref="karafCellarClusterManager" />
+ <property name="karafCellarEventProducer" ref="karafCellarEventProducer" />
+ <property name="karafCellarGroupManager" ref="karafCellarGroupManager" />
+ <property name="karafCellarGroupName" value="${cluster.group}" />
+ <property name="osgiConfigurationAdmin" ref="osgiConfigurationAdmin" />
+ <property name="karafJMXUsername" value="${cluster.jmxUsername}" />
+ <property name="karafJMXPassword" value="${cluster.jmxPassword}" />
+ <property name="karafJMXPort" value="${cluster.jmxPort}" />
+ </bean>
+ <service id="clusterService" ref="clusterServiceImpl" auto-export="interfaces"/>
+
<!-- We use a listener here because using the list directly for listening to proxies coming from the same bundle didn't seem to work -->
<reference-list id="eventListenerServices"
interface="org.apache.unomi.api.services.EventListenerService"
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/services/src/main/resources/hazelcast.xml
----------------------------------------------------------------------
diff --git a/services/src/main/resources/hazelcast.xml b/services/src/main/resources/hazelcast.xml
new file mode 100644
index 0000000..0fc6f5d
--- /dev/null
+++ b/services/src/main/resources/hazelcast.xml
@@ -0,0 +1,219 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<hazelcast xsi:schemaLocation="http://www.hazelcast.com/schema/config hazelcast-config-3.2.xsd"
+ xmlns="http://www.hazelcast.com/schema/config"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <group>
+ <name>cellar</name>
+ <password>pass</password>
+ </group>
+ <management-center enabled="false">http://localhost:8080/mancenter</management-center>
+ <network>
+ <port auto-increment="true" port-count="100">5701</port>
+ <outbound-ports>
+ <!--
+ Allowed port range when connecting to other nodes.
+ 0 or * means use system provided port.
+ -->
+ <ports>0</ports>
+ </outbound-ports>
+ <join>
+ <multicast enabled="false">
+ <multicast-group>224.2.2.3</multicast-group>
+ <multicast-port>54327</multicast-port>
+ </multicast>
+ <tcp-ip enabled="true">
+ <interface>127.0.0.1</interface>
+ </tcp-ip>
+ <aws enabled="false">
+ <access-key>my-access-key</access-key>
+ <secret-key>my-secret-key</secret-key>
+ <!--optional, default is us-east-1 -->
+ <region>us-west-1</region>
+ <!--optional, default is ec2.amazonaws.com. If set, region shouldn't be set as it will override this property -->
+ <host-header>ec2.amazonaws.com</host-header>
+ <!-- optional, only instances belonging to this group will be discovered, default will try all running instances -->
+ <security-group-name>hazelcast-sg</security-group-name>
+ <tag-key>type</tag-key>
+ <tag-value>hz-nodes</tag-value>
+ </aws>
+ </join>
+ <interfaces enabled="false">
+ <interface>10.10.1.*</interface>
+ </interfaces>
+ <ssl enabled="false"/>
+ <socket-interceptor enabled="false"/>
+ <symmetric-encryption enabled="false">
+ <!--
+ encryption algorithm such as
+ DES/ECB/PKCS5Padding,
+ PBEWithMD5AndDES,
+ AES/CBC/PKCS5Padding,
+ Blowfish,
+ DESede
+ -->
+ <algorithm>PBEWithMD5AndDES</algorithm>
+ <!-- salt value to use when generating the secret key -->
+ <salt>thesalt</salt>
+ <!-- pass phrase to use when generating the secret key -->
+ <password>thepass</password>
+ <!-- iteration count to use when generating the secret key -->
+ <iteration-count>19</iteration-count>
+ </symmetric-encryption>
+ </network>
+ <partition-group enabled="false"/>
+ <executor-service>
+ <pool-size>16</pool-size>
+ <!-- Queue capacity. 0 means Integer.MAX_VALUE -->
+ <queue-capacity>0</queue-capacity>
+ </executor-service>
+ <queue name="default">
+ <!--
+ Maximum size of the queue. When a JVM's local queue size reaches the maximum,
+ all put/offer operations will get blocked until the queue size
+ of the JVM goes down below the maximum.
+ Any integer between 0 and Integer.MAX_VALUE. 0 means
+ Integer.MAX_VALUE. Default is 0.
+ -->
+ <max-size>0</max-size>
+ <!--
+ Number of backups. If 1 is set as the backup-count for example,
+ then all entries of the map will be copied to another JVM for
+ fail-safety. 0 means no backup.
+ -->
+ <backup-count>1</backup-count>
+ <!--
+ Number of async backups. 0 means no backup.
+ -->
+ <async-backup-count>0</async-backup-count>
+ <empty-queue-ttl>-1</empty-queue-ttl>
+ </queue>
+
+ <map name="default">
+ <!--
+ Data type that will be used for storing recordMap.
+ Possible values:
+ BINARY (default): keys and values will be stored as binary data
+ OBJECT : values will be stored in their object forms
+ OFFHEAP : values will be stored in non-heap region of JVM
+ -->
+ <in-memory-format>BINARY</in-memory-format>
+ <!--
+ Number of backups. If 1 is set as the backup-count for example,
+ then all entries of the map will be copied to another JVM for
+ fail-safety. 0 means no backup.
+ -->
+ <backup-count>1</backup-count>
+ <!--
+ Number of async backups. 0 means no backup.
+ -->
+ <async-backup-count>0</async-backup-count>
+ <!--
+ Maximum number of seconds for each entry to stay in the map. Entries that are
+ older than <time-to-live-seconds> and not updated for <time-to-live-seconds>
+ will get automatically evicted from the map.
+ Any integer between 0 and Integer.MAX_VALUE. 0 means infinite. Default is 0.
+ -->
+ <time-to-live-seconds>0</time-to-live-seconds>
+ <!--
+ Maximum number of seconds for each entry to stay idle in the map. Entries that are
+ idle(not touched) for more than <max-idle-seconds> will get
+ automatically evicted from the map. Entry is touched if get, put or containsKey is called.
+ Any integer between 0 and Integer.MAX_VALUE. 0 means infinite. Default is 0.
+ -->
+ <max-idle-seconds>0</max-idle-seconds>
+ <!--
+ Valid values are:
+ NONE (no eviction),
+ LRU (Least Recently Used),
+ LFU (Least Frequently Used).
+ NONE is the default.
+ -->
+ <eviction-policy>NONE</eviction-policy>
+ <!--
+ Maximum size of the map. When max size is reached,
+ map is evicted based on the policy defined.
+ Any integer between 0 and Integer.MAX_VALUE. 0 means
+ Integer.MAX_VALUE. Default is 0.
+ -->
+ <max-size policy="PER_NODE">0</max-size>
+ <!--
+ When max. size is reached, specified percentage of
+ the map will be evicted. Any integer between 0 and 100.
+ If 25 is set for example, 25% of the entries will
+ get evicted.
+ -->
+ <eviction-percentage>25</eviction-percentage>
+ <!--
+ While recovering from split-brain (network partitioning),
+ map entries in the small cluster will merge into the bigger cluster
+ based on the policy set here. When an entry merge into the
+ cluster, there might an existing entry with the same key already.
+ Values of these entries might be different for that same key.
+ Which value should be set for the key? Conflict is resolved by
+ the policy set here. Default policy is PutIfAbsentMapMergePolicy
+
+ There are built-in merge policies such as
+ com.hazelcast.map.merge.PassThroughMergePolicy; entry will be added if there is no existing entry for the key.
+ com.hazelcast.map.merge.PutIfAbsentMapMergePolicy ; entry will be added if the merging entry doesn't exist in the cluster.
+ com.hazelcast.map.merge.HigherHitsMapMergePolicy ; entry with the higher hits wins.
+ com.hazelcast.map.merge.LatestUpdateMapMergePolicy ; entry with the latest update wins.
+ -->
+ <merge-policy>com.hazelcast.map.merge.PassThroughMergePolicy</merge-policy>
+ </map>
+
+ <multimap name="default">
+ <backup-count>1</backup-count>
+ <value-collection-type>SET</value-collection-type>
+ </multimap>
+
+ <multimap name="default">
+ <backup-count>1</backup-count>
+ <value-collection-type>SET</value-collection-type>
+ </multimap>
+
+ <list name="default">
+ <backup-count>1</backup-count>
+ </list>
+
+ <set name="default">
+ <backup-count>1</backup-count>
+ </set>
+
+ <jobtracker name="default">
+ <max-thread-size>0</max-thread-size>
+ <!-- Queue size 0 means number of partitions * 2 -->
+ <queue-size>0</queue-size>
+ <retry-count>0</retry-count>
+ <chunk-size>1000</chunk-size>
+ <communicate-stats>true</communicate-stats>
+ <topology-changed-strategy>CANCEL_RUNNING_OPERATION</topology-changed-strategy>
+ </jobtracker>
+
+ <semaphore name="default">
+ <initial-permits>0</initial-permits>
+ <backup-count>1</backup-count>
+ <async-backup-count>0</async-backup-count>
+ </semaphore>
+
+ <serialization>
+ <portable-version>0</portable-version>
+ </serialization>
+
+ <services enable-defaults="true" />
+</hazelcast>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/services/src/main/resources/org.apache.unomi.cluster.cfg
----------------------------------------------------------------------
diff --git a/services/src/main/resources/org.apache.unomi.cluster.cfg b/services/src/main/resources/org.apache.unomi.cluster.cfg
new file mode 100644
index 0000000..a9ecf2e
--- /dev/null
+++ b/services/src/main/resources/org.apache.unomi.cluster.cfg
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+cluster.group=default
+cluster.jmxUsername=karaf
+cluster.jmxPassword=karaf
+cluster.jmxPort=1099
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/services/src/main/resources/org.apache.unomi.services.cfg
----------------------------------------------------------------------
diff --git a/services/src/main/resources/org.apache.unomi.services.cfg b/services/src/main/resources/org.apache.unomi.services.cfg
index d69da60..9e59210 100644
--- a/services/src/main/resources/org.apache.unomi.services.cfg
+++ b/services/src/main/resources/org.apache.unomi.services.cfg
@@ -19,7 +19,7 @@
profile.purge.interval=1
# Purge profiles that have been inactive for a specific number of days
-profile.purge.inactiveTime=30
+profile.purge.inactiveTime=180
# Purge profiles that have been created for a specific number of days
profile.purge.existTime=-1
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/src/site/markdown/clustering.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/clustering.md b/src/site/markdown/clustering.md
index 7c51889..9e1a548 100644
--- a/src/site/markdown/clustering.md
+++ b/src/site/markdown/clustering.md
@@ -18,30 +18,21 @@
Cluster setup
=============
-Context server relies on Elasticsearch to discover and configure its cluster. You just need to install multiple context
-servers on the same network, and enable the discovery protocol in $MY_KARAF_HOME/etc/org.apache.unomi.persistence.elasticsearch.cfg file :
-
- discovery.zen.ping.multicast.enabled=true
+Context server relies on Apache Karaf Cellar to discover and configure its cluster. You just need to install multiple context
+servers on the same network, and enable the discovery protocol in $MY_KARAF_HOME/etc/hazelcast.xml file.
All nodes on the same network, sharing the same cluster name will be part of the same cluster.
-###Recommended configurations
-
-It is recommended to have one node dedicated to the context server, where the other nodes take care of the
-Elasticsearch persistence. The node dedicated to the context server will have node.data set to false.
-
#### 2 nodes configuration
One node dedicated to context server, 1 node for elasticsearch storage.
Node A :
- node.data=true
numberOfReplicas=0
monthlyIndex.numberOfReplicas=0
Node B :
- node.data=false
numberOfReplicas=0
monthlyIndex.numberOfReplicas=0
@@ -50,34 +41,15 @@ One node dedicated to context server, 2 nodes for elasticsearch storage with fau
Node A :
- node.data=false
numberOfReplicas=1
monthlyIndex.numberOfReplicas=1
Node B :
- node.data=true
numberOfReplicas=1
monthlyIndex.numberOfReplicas=1
Node C :
- node.data=true
numberOfReplicas=1
monthlyIndex.numberOfReplicas=1
-
-### Specific configuration
-If multicast is not allowed on your network, you'll need to switch to unicast protocol and manually configure the server IPs. This can be
-done by disabling the elasticsearch automatic discovery in $MY_KARAF_HOME/etc/org.apache.unomi.persistence.elasticsearch.cfg :
-
- discovery.zen.ping.multicast.enabled=false
-
-
-And then set the property discovery.zen.ping.unicast.hosts in $MY_KARAF_HOME/etc/elasticsearch.yml files :
-
-
- discovery.zen.ping.unicast.hosts: [\u2018192.168.0.1:9300', \u2018192.168.0.2:9300']
-
-
-More information and configuration options can be found at :
-[https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html)
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/src/site/markdown/configuration.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/configuration.md b/src/site/markdown/configuration.md
index b925d48..0bf595c 100644
--- a/src/site/markdown/configuration.md
+++ b/src/site/markdown/configuration.md
@@ -43,15 +43,6 @@ with the following contents:
cluster.name=contextElasticSearch
index.name=context
- elasticSearchConfig=file:${karaf.etc}/elasticsearch.yml
-
-And replace the cluster.name parameter here by your cluster name.
-
-You can also put an elasticsearch configuration file in $MY_KARAF_HOME/etc/elasticsearch.yml ,
-and put any standard Elasticsearch configuration options in this last file.
-
-If you want your context server to be a client only on a cluster of elasticsearch nodes, just set the node.data property
-to false.
Installing the MaxMind GeoIPLite2 IP lookup database
----------------------------------------------------
@@ -160,23 +151,7 @@ node-to-node communication : 9200 (Elasticsearch REST API), 9300 (Elasticsearch
Of course any ports listed here are the default ports configured in each server, you may adjust them if needed.
-Step 2 : Adjust the Context Server IP filtering
-
-By default the Context Server limits to connections to port 9200 and 9300 to the following IP ranges
-
- - localhost
- - 127.0.0.1
- - ::1
- - the current subnet (i.e., 192.168.1.0-192.168.1.255)
-
-(this is done using a custom plugin for Elasticsearch, that you may find here :
-https://git-wip-us.apache.org/repos/asf/incubator-unomi/context-server/persistence-elasticsearch/plugins/security)
-
-You can adjust this setting by using the following setting in the $MY_KARAF_HOME/etc/elasticsearch.yml file :
-
- security.ipranges: localhost,127.0.0.1,::1,10.0.1.0-10.0.1.255
-
-Step 3 : Follow industry recommended best practices for securing Elasticsearch
+Step 2 : Follow industry recommended best practices for securing Elasticsearch
You may find more valuable recommendations here :
[35/50] [abbrv] incubator-unomi git commit: Update feature to add
dependency to cellar
Posted by sh...@apache.org.
Update feature to add dependency to cellar
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/69bb3cc9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/69bb3cc9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/69bb3cc9
Branch: refs/heads/master
Commit: 69bb3cc9bb8af844c7e1922e1bd8eaeefdf0ce31
Parents: 207c2cc
Author: Serge Huber <sh...@apache.org>
Authored: Wed Dec 21 20:19:55 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Wed Dec 21 20:19:55 2016 +0100
----------------------------------------------------------------------
kar/pom.xml | 31 +++++++------------------------
1 file changed, 7 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/69bb3cc9/kar/pom.xml
----------------------------------------------------------------------
diff --git a/kar/pom.xml b/kar/pom.xml
index cee6add..8a1c1c4 100644
--- a/kar/pom.xml
+++ b/kar/pom.xml
@@ -108,6 +108,13 @@
<groupId>org.apache.servicemix.bundles</groupId>
<artifactId>org.apache.servicemix.bundles.joda-time</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.karaf.cellar</groupId>
+ <artifactId>apache-karaf-cellar</artifactId>
+ <classifier>features</classifier>
+ <type>xml</type>
+ <scope>runtime</scope>
+ </dependency>
</dependencies>
<build>
@@ -132,30 +139,6 @@
<startLevel>85</startLevel>
</configuration>
</plugin>
- <!--
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>attach-artifacts</id>
- <phase>package</phase>
- <goals>
- <goal>attach-artifact</goal>
- </goals>
- <configuration>
- <artifacts>
- <artifact>
- <file>target/feature/feature.xml</file>
- <classifier>features</classifier>
- <type>xml</type>
- </artifact>
- </artifacts>
- </configuration>
- </execution>
- </executions>
- </plugin>
- -->
</plugins>
</build>
[50/50] [abbrv] incubator-unomi git commit:
[jgitflow-maven-plugin]merging 'feature-UNOMI-70-ES5X' into 'master'
Posted by sh...@apache.org.
[jgitflow-maven-plugin]merging 'feature-UNOMI-70-ES5X' into 'master'
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/b1c88c4c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/b1c88c4c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/b1c88c4c
Branch: refs/heads/master
Commit: b1c88c4c4c7ba91090a496e8b9f2aa7ec1ecaa61
Parents: a4955ca 544d2d4
Author: Serge Huber <sh...@apache.org>
Authored: Mon Jan 9 17:11:30 2017 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Mon Jan 9 17:11:30 2017 +0100
----------------------------------------------------------------------
.gitignore | 3 +
README.md | 71 +-
.../java/org/apache/unomi/api/PartialList.java | 25 +
.../apache/unomi/api/conditions/Condition.java | 10 +
.../unomi/api/services/ClusterService.java | 1 -
buildAndRunNoTests.sh | 42 +
extensions/geonames/rest/pom.xml | 5 -
.../unomi/geonames/services/GeonameEntry.java | 4 +-
.../geonames/services/GeonamesServiceImpl.java | 101 +-
.../META-INF/cxs/mappings/geonameEntry.json | 62 +-
extensions/lists-extension/rest/pom.xml | 5 -
.../META-INF/cxs/mappings/userList.json | 36 -
extensions/privacy-extension/rest/pom.xml | 5 -
itests/pom.xml | 55 +-
.../java/org/apache/unomi/itests/AllITs.java | 38 +
.../java/org/apache/unomi/itests/AllTests.java | 37 -
.../java/org/apache/unomi/itests/BaseIT.java | 108 +
.../java/org/apache/unomi/itests/BaseTest.java | 100 -
.../java/org/apache/unomi/itests/BasicIT.java | 99 +
.../java/org/apache/unomi/itests/BasicTest.java | 99 -
.../unomi/itests/ConditionESQueryBuilderIT.java | 59 +
.../itests/ConditionESQueryBuilderTest.java | 59 -
.../unomi/itests/ConditionEvaluatorIT.java | 178 +
.../unomi/itests/ConditionEvaluatorTest.java | 178 -
.../apache/unomi/itests/ProfileServiceIT.java | 53 +
.../java/org/apache/unomi/itests/SegmentIT.java | 48 +
.../org/apache/unomi/itests/SegmentTest.java | 48 -
kar/pom.xml | 31 +-
kar/src/main/feature/feature.xml | 18 +-
package/pom.xml | 51 +-
package/src/main/resources/bin/setenv | 2 -
package/src/main/resources/bin/setenv.bat | 1 -
persistence-elasticsearch/core/pom.xml | 256 +-
.../ElasticSearchPersistenceServiceImpl.java | 1002 +++--
.../conditions/ConditionContextHelper.java | 35 +-
.../conditions/ConditionESQueryBuilder.java | 4 +-
.../ConditionESQueryBuilderDispatcher.java | 42 +-
.../ConditionEvaluatorDispatcher.java | 24 +-
.../META-INF/cxs/mappings/_default_.json | 22 +
.../META-INF/cxs/mappings/actionType.json | 16 -
.../META-INF/cxs/mappings/campaign.json | 58 +-
.../META-INF/cxs/mappings/campaignevent.json | 51 +-
.../META-INF/cxs/mappings/conditionType.json | 16 -
.../resources/META-INF/cxs/mappings/event.json | 39 +-
.../resources/META-INF/cxs/mappings/goal.json | 45 +-
.../META-INF/cxs/mappings/persona.json | 16 -
.../META-INF/cxs/mappings/personaSession.json | 27 +-
.../META-INF/cxs/mappings/profile.json | 39 +-
.../META-INF/cxs/mappings/propertyType.json | 68 -
.../resources/META-INF/cxs/mappings/rule.json | 36 -
.../META-INF/cxs/mappings/scoring.json | 41 +-
.../META-INF/cxs/mappings/segment.json | 38 +-
.../META-INF/cxs/mappings/session.json | 40 +-
.../resources/OSGI-INF/blueprint/blueprint.xml | 53 +-
.../core/src/main/resources/elasticsearch.yml | 379 --
.../core/src/main/resources/log4j2.xml | 29 +
.../src/main/resources/mapping-FoldToASCII.txt | 3813 ++++++++++++++++++
...g.apache.unomi.persistence.elasticsearch.cfg | 17 +-
.../plugin/security/SecurityPlugin.java | 38 +-
.../plugin/security/SecurityPluginModule.java | 3 +-
.../plugin/security/SecurityPluginService.java | 11 +-
.../persistence/spi/PersistenceService.java | 77 +-
.../BooleanConditionESQueryBuilder.java | 34 +-
...onByPointSessionConditionESQueryBuilder.java | 16 +-
.../MatchAllConditionESQueryBuilder.java | 8 +-
.../conditions/NotConditionESQueryBuilder.java | 8 +-
.../PastEventConditionESQueryBuilder.java | 8 +-
.../PropertyConditionESQueryBuilder.java | 66 +-
.../conditions/PropertyConditionEvaluator.java | 12 +-
...rceEventPropertyConditionESQueryBuilder.java | 27 +-
.../HoverEventConditionESQueryBuilder.java | 23 +-
pom.xml | 46 +-
rest/pom.xml | 5 -
services/pom.xml | 32 +-
.../actions/ActionExecutorDispatcher.java | 24 +-
.../services/services/ClusterServiceImpl.java | 304 ++
.../services/services/GoalsServiceImpl.java | 13 +-
.../services/services/RulesServiceImpl.java | 37 +-
.../services/services/SegmentServiceImpl.java | 99 +-
.../resources/OSGI-INF/blueprint/blueprint.xml | 55 +-
services/src/main/resources/hazelcast.xml | 219 +
.../main/resources/org.apache.unomi.cluster.cfg | 20 +
.../resources/org.apache.unomi.services.cfg | 6 +-
src/site/markdown/clustering.md | 32 +-
src/site/markdown/configuration.md | 27 +-
85 files changed, 6551 insertions(+), 2537 deletions(-)
----------------------------------------------------------------------
[18/50] [abbrv] incubator-unomi git commit: Merge branch
'feature-UNOMI-28-ES2X' into feature-UNOMI-70-ES5X
Posted by sh...@apache.org.
Merge branch 'feature-UNOMI-28-ES2X' into feature-UNOMI-70-ES5X
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/a217ba01
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/a217ba01
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/a217ba01
Branch: refs/heads/master
Commit: a217ba015da29167bf3209e4a792349fe21dcc17
Parents: 96a018e d7a7969
Author: Serge Huber <sh...@apache.org>
Authored: Tue Dec 6 13:46:20 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Tue Dec 6 13:46:20 2016 +0100
----------------------------------------------------------------------
.../services/services/SegmentServiceImpl.java | 26 +++++++++++++++++---
1 file changed, 22 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[16/50] [abbrv] incubator-unomi git commit: UNOMI 70 - Upgrade to
ElasticSearch 5.0 - Upgrade to ElasticSearch 5.0.1 - Upgrade CXF to 3.0.2 -
Upgrade to Karaf 3.0.8 - Replace Unomi clustering from ES clustering to Karaf
Cellar clustering
Posted by sh...@apache.org.
UNOMI 70 - Upgrade to ElasticSearch 5.0
- Upgrade to ElasticSearch 5.0.1
- Upgrade CXF to 3.0.2
- Upgrade to Karaf 3.0.8
- Replace Unomi clustering from ES clustering to Karaf Cellar clustering
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/96a018ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/96a018ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/96a018ec
Branch: refs/heads/master
Commit: 96a018ecb1b5b907930504e69f6da9357db8385e
Parents: 90eb893
Author: Serge Huber <sh...@apache.org>
Authored: Mon Dec 5 10:43:29 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Mon Dec 5 10:43:29 2016 +0100
----------------------------------------------------------------------
extensions/geonames/rest/pom.xml | 5 -
.../META-INF/cxs/mappings/geonameEntry.json | 42 +-
extensions/lists-extension/rest/pom.xml | 5 -
.../META-INF/cxs/mappings/userList.json | 23 +-
extensions/privacy-extension/rest/pom.xml | 5 -
itests/pom.xml | 41 +-
kar/src/main/feature/feature.xml | 2 +-
package/pom.xml | 36 +-
persistence-elasticsearch/core/pom.xml | 115 +++--
.../ElasticSearchPersistenceServiceImpl.java | 490 ++++++++++---------
.../META-INF/cxs/mappings/actionType.json | 4 +-
.../META-INF/cxs/mappings/campaign.json | 32 +-
.../META-INF/cxs/mappings/campaignevent.json | 30 +-
.../META-INF/cxs/mappings/conditionType.json | 66 ++-
.../resources/META-INF/cxs/mappings/event.json | 319 +++++++++++-
.../resources/META-INF/cxs/mappings/goal.json | 28 +-
.../META-INF/cxs/mappings/persona.json | 22 +-
.../META-INF/cxs/mappings/personaSession.json | 81 ++-
.../META-INF/cxs/mappings/profile.json | 42 +-
.../META-INF/cxs/mappings/propertyType.json | 45 +-
.../resources/META-INF/cxs/mappings/rule.json | 21 +-
.../META-INF/cxs/mappings/scoring.json | 25 +-
.../META-INF/cxs/mappings/segment.json | 21 +-
.../META-INF/cxs/mappings/session.json | 35 +-
.../resources/OSGI-INF/blueprint/blueprint.xml | 26 +-
.../core/src/main/resources/log4j2.xml | 29 ++
...g.apache.unomi.persistence.elasticsearch.cfg | 7 +-
.../plugin/security/SecurityPlugin.java | 34 +-
.../plugin/security/SecurityPluginService.java | 7 +-
.../persistence/spi/PersistenceService.java | 8 -
.../BooleanConditionESQueryBuilder.java | 18 +-
...onByPointSessionConditionESQueryBuilder.java | 6 +-
.../conditions/PropertyConditionEvaluator.java | 4 +-
pom.xml | 31 +-
rest/pom.xml | 5 -
services/pom.xml | 10 -
36 files changed, 1132 insertions(+), 588 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/extensions/geonames/rest/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/geonames/rest/pom.xml b/extensions/geonames/rest/pom.xml
index 72a8e74..ff52075 100644
--- a/extensions/geonames/rest/pom.xml
+++ b/extensions/geonames/rest/pom.xml
@@ -52,11 +52,6 @@
<dependency>
<groupId>org.apache.cxf</groupId>
- <artifactId>cxf-rt-core</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.cxf</groupId>
<artifactId>cxf-rt-frontend-jaxws</artifactId>
<scope>provided</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json
----------------------------------------------------------------------
diff --git a/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json b/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json
index 63664fe..612f3ca 100644
--- a/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json
+++ b/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json
@@ -2,56 +2,46 @@
"geonameEntry": {
"properties": {
"admin1Code": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"admin2Code": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"admin3Code": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"admin4Code": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"alternatenames": {
- "type": "string"
+ "type": "keyword"
},
"asciiname": {
- "type": "string"
+ "type": "keyword"
},
"cc2": {
- "type": "string"
+ "type": "keyword"
},
"countryCode": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"dem": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"elevation": {
"type": "long"
},
"featureClass": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"featureCode": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"itemId": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"itemType": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"location": {
"type": "geo_point"
@@ -61,15 +51,13 @@
"format": "strict_date_optional_time||epoch_millis"
},
"name": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"population": {
"type": "long"
},
"timezone": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/extensions/lists-extension/rest/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/lists-extension/rest/pom.xml b/extensions/lists-extension/rest/pom.xml
index 0988a06..f2d31ad 100644
--- a/extensions/lists-extension/rest/pom.xml
+++ b/extensions/lists-extension/rest/pom.xml
@@ -53,11 +53,6 @@
<dependency>
<groupId>org.apache.cxf</groupId>
- <artifactId>cxf-rt-core</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.cxf</groupId>
<artifactId>cxf-rt-frontend-jaxws</artifactId>
<scope>provided</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json
----------------------------------------------------------------------
diff --git a/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json b/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json
index 3bb8ae0..244d71a 100644
--- a/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json
+++ b/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json
@@ -4,27 +4,25 @@
{
"all": {
"match": "*",
- "match_mapping_type": "string",
+ "match_mapping_type": "text",
"mapping": {
- "type": "string",
- "analyzer": "folding"
+ "type": "text",
+ "analyzer" : "folding"
}
}
}
],
"properties": {
"itemId": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"itemType": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"metadata": {
"properties": {
"description": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
},
"enabled": {
@@ -34,22 +32,19 @@
"type": "boolean"
},
"id": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"missingPlugins": {
"type": "boolean"
},
"name": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"readOnly": {
"type": "boolean"
},
"scope": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/extensions/privacy-extension/rest/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/privacy-extension/rest/pom.xml b/extensions/privacy-extension/rest/pom.xml
index 2bc7560..7c51644 100644
--- a/extensions/privacy-extension/rest/pom.xml
+++ b/extensions/privacy-extension/rest/pom.xml
@@ -52,11 +52,6 @@
<dependency>
<groupId>org.apache.cxf</groupId>
- <artifactId>cxf-rt-core</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.cxf</groupId>
<artifactId>cxf-rt-frontend-jaxws</artifactId>
<scope>provided</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/itests/pom.xml
----------------------------------------------------------------------
diff --git a/itests/pom.xml b/itests/pom.xml
index 285a2f6..11e0a94 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -134,13 +134,52 @@
</executions>
</plugin>
<plugin>
+ <groupId>com.github.alexcojocaru</groupId>
+ <artifactId>elasticsearch-maven-plugin</artifactId>
+ <!-- REPLACE THE FOLLOWING WITH THE PLUGIN VERSION YOU NEED -->
+ <version>5.0</version>
+ <configuration>
+ <clusterName>contextElasticSearch</clusterName>
+ <tcpPort>9300</tcpPort>
+ <httpPort>9200</httpPort>
+ </configuration>
+ <executions>
+ <!--
+ The elasticsearch maven plugin goals are by default bound to the
+ pre-integration-test and post-integration-test phases
+ -->
+ <execution>
+ <id>start-elasticsearch</id>
+ <phase>pre-integration-test</phase>
+ <goals>
+ <goal>runforked</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>stop-elasticsearch</id>
+ <phase>post-integration-test</phase>
+ <goals>
+ <goal>stop</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
+ <artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<includes>
<include>**/*AllTests.java</include>
</includes>
</configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
</plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/kar/src/main/feature/feature.xml
----------------------------------------------------------------------
diff --git a/kar/src/main/feature/feature.xml b/kar/src/main/feature/feature.xml
index 5517a19..39413fb 100644
--- a/kar/src/main/feature/feature.xml
+++ b/kar/src/main/feature/feature.xml
@@ -20,7 +20,7 @@
<feature description="unomi-kar" version="${project.version}" name="unomi-kar"
start-level="75">
<feature>war</feature>
- <feature version="[2.7.11,3)">cxf</feature>
+ <feature>cxf</feature>
<feature>openwebbeans</feature>
<feature>pax-cdi-web-openwebbeans</feature>
<configfile finalname="/etc/org.apache.unomi.web.cfg">mvn:org.apache.unomi/unomi-wab/${project.version}/cfg/unomicfg</configfile>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/package/pom.xml
----------------------------------------------------------------------
diff --git a/package/pom.xml b/package/pom.xml
index 4a39217..6f622d9 100644
--- a/package/pom.xml
+++ b/package/pom.xml
@@ -64,14 +64,23 @@
<scope>runtime</scope>
</dependency>
+ <!--
<dependency>
- <!-- scope is runtime so the feature repo is listed in the features service config file, and features may be installed using the karaf-maven-plugin configuration -->
<groupId>org.apache.cxf.karaf</groupId>
<artifactId>apache-cxf</artifactId>
<classifier>features</classifier>
<type>xml</type>
<scope>runtime</scope>
</dependency>
+ -->
+ <dependency>
+ <groupId>org.apache.karaf.cellar</groupId>
+ <artifactId>apache-karaf-cellar</artifactId>
+ <classifier>features</classifier>
+ <type>xml</type>
+ <scope>runtime</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-kar</artifactId>
@@ -167,30 +176,8 @@
<artifactItems>
<artifactItem>
<groupId>org.elasticsearch.plugin</groupId>
- <artifactId>delete-by-query</artifactId>
- <version>${elasticsearch.version}</version>
- <type>zip</type>
- <outputDirectory>${project.build.directory}/assembly/elasticsearch/plugins/delete-by-query</outputDirectory>
- </artifactItem>
- <artifactItem>
- <groupId>org.elasticsearch.module</groupId>
- <artifactId>lang-groovy</artifactId>
- <version>${elasticsearch.version}</version>
- <type>zip</type>
- <outputDirectory>${project.build.directory}/assembly/elasticsearch/modules/lang-groovy</outputDirectory>
- </artifactItem>
- <artifactItem>
- <groupId>org.elasticsearch.module</groupId>
- <artifactId>lang-expression</artifactId>
- <version>${elasticsearch.version}</version>
- <type>zip</type>
- <outputDirectory>${project.build.directory}/assembly/elasticsearch/modules/lang-expression</outputDirectory>
- </artifactItem>
- <artifactItem>
- <groupId>org.elasticsearch.module</groupId>
- <artifactId>reindex</artifactId>
+ <artifactId>reindex-client</artifactId>
<version>${elasticsearch.version}</version>
- <type>zip</type>
<outputDirectory>${project.build.directory}/assembly/elasticsearch/modules/reindex</outputDirectory>
</artifactItem>
</artifactItems>
@@ -299,6 +286,7 @@
<feature>cxf</feature>
<feature>openwebbeans</feature>
<feature>pax-cdi-web-openwebbeans</feature>
+ <feature>cellar</feature>
<feature>unomi-kar</feature>
</bootFeatures>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/pom.xml b/persistence-elasticsearch/core/pom.xml
index 6380219..6a05775 100644
--- a/persistence-elasticsearch/core/pom.xml
+++ b/persistence-elasticsearch/core/pom.xml
@@ -35,6 +35,11 @@
<artifactId>org.osgi.core</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.osgi</groupId>
+ <artifactId>org.osgi.compendium</artifactId>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
@@ -49,14 +54,29 @@
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>org.apache.karaf.cellar</groupId>
+ <artifactId>org.apache.karaf.cellar.core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.karaf.cellar</groupId>
+ <artifactId>org.apache.karaf.cellar.config</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>20.0</version>
+ </dependency>
+ <dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>${elasticsearch.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.elasticsearch.plugin</groupId>
- <artifactId>delete-by-query</artifactId>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>transport</artifactId>
<version>${elasticsearch.version}</version>
<scope>compile</scope>
</dependency>
@@ -64,41 +84,43 @@
<!-- The following are optional dependencies from the ElasticSearch that are made mandatory -->
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>log4j-over-slf4j</artifactId>
- <version>1.7.7</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>net.java.dev.jna</groupId>
- <artifactId>jna</artifactId>
- <version>4.1.0</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>com.github.spullara.mustache.java</groupId>
- <artifactId>compiler</artifactId>
- <version>0.8.13</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>2.6.2</version>
<scope>compile</scope>
+ <optional>true</optional>
</dependency>
<dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>2.8.1</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>2.6.2</version>
<scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </exclusion>
+ </exclusions>
+ <optional>true</optional>
</dependency>
<dependency>
- <groupId>org.noggit</groupId>
- <artifactId>noggit</artifactId>
- <version>0.7</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ <version>2.6.2</version>
<scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </exclusion>
+ </exclusions>
+ <optional>true</optional>
</dependency>
+
<!-- End of optional ElasticSearch dependencies -->
<dependency>
@@ -169,6 +191,39 @@
sun.misc;resolution:=optional,
sun.security.util;resolution:=optional,
sun.security.x509;resolution:=optional,
+ com.google.protobuf.nano;resolution:=optional,
+ com.jcraft.jzlib;resolution:=optional,
+ com.ning.compress;resolution:=optional,
+ com.ning.compress.lzf;resolution:=optional,
+ com.ning.compress.lzf.util;resolution:=optional,
+ javassist;resolution:=optional,
+ lzma.sdk;resolution:=optional,
+ lzma.sdk.lzma;resolution:=optional,
+ net.jpountz.lz4;resolution:=optional,
+ net.jpountz.xxhash;resolution:=optional,
+ org.apache.tomcat;resolution:=optional,
+ org.eclipse.jetty.alpn;resolution:=optional,
+ org.joda.convert;resolution:=optional,
+ org.locationtech.spatial4j.context;resolution:=optional,
+ org.locationtech.spatial4j.context.jts;resolution:=optional,
+ org.locationtech.spatial4j.distance;resolution:=optional,
+ org.locationtech.spatial4j.exception;resolution:=optional,
+ org.locationtech.spatial4j.io;resolution:=optional,
+ org.locationtech.spatial4j.shape;resolution:=optional,
+ org.locationtech.spatial4j.shape.impl;resolution:=optional,
+ org.locationtech.spatial4j.shape.jts;resolution:=optional,
+ org.zeromq;resolution:=optional,
+ org.apache.commons.compress.compressors;resolution:=optional,
+ org.apache.commons.compress.utils;resolution:=optional,
+ org.apache.commons.csv;resolution:=optional,
+ org.apache.kafka.clients.producer;resolution:=optional,
+ javax.persistence;resolution:=optional,
+ com.google.errorprone.annotations.concurrent;resolution:=optional,
+ com.lmax.disruptor;resolution:=optional,
+ com.lmax.disruptor.dsl;resolution:=optional,
+ com.fasterxml.jackson.dataformat.xml;resolution:=optional,
+ com.fasterxml.jackson.dataformat.xml.annotation;resolution:=optional,
+ com.fasterxml.jackson.dataformat.xml.util;resolution:=optional,
*
</Import-Package>
<Export-Package>
@@ -176,7 +231,7 @@
org.elasticsearch.index.query.*;version="${elasticsearch.version}",
org.apache.unomi.persistence.elasticsearch.conditions;version="${project.version}"
</Export-Package>
- <Embed-Dependency>*;scope=compile|runtime;artifactId=!log4j</Embed-Dependency>
+ <Embed-Dependency>*;scope=compile|runtime</Embed-Dependency>
<Embed-Transitive>true</Embed-Transitive>
</instructions>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 794b03b..0dc3899 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -17,7 +17,13 @@
package org.apache.unomi.persistence.elasticsearch;
-import com.google.common.collect.UnmodifiableIterator;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.karaf.cellar.config.ClusterConfigurationEvent;
+import org.apache.karaf.cellar.config.Constants;
+import org.apache.karaf.cellar.core.*;
+import org.apache.karaf.cellar.core.control.SwitchStatus;
+import org.apache.karaf.cellar.core.event.EventProducer;
+import org.apache.karaf.cellar.core.event.EventType;
import org.apache.unomi.api.ClusterNode;
import org.apache.unomi.api.Item;
import org.apache.unomi.api.PartialList;
@@ -31,33 +37,22 @@ import org.apache.unomi.persistence.elasticsearch.conditions.*;
import org.apache.unomi.persistence.spi.CustomObjectMapper;
import org.apache.unomi.persistence.spi.PersistenceService;
import org.apache.unomi.persistence.spi.aggregate.*;
-import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
-import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
-import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
-import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
-import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.bulk.*;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryAction;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.percolate.PercolateResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
-import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
@@ -67,7 +62,6 @@ import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
-import org.elasticsearch.node.Node;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.SearchHit;
@@ -77,91 +71,86 @@ import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregation;
import org.elasticsearch.search.aggregations.bucket.filter.Filter;
import org.elasticsearch.search.aggregations.bucket.global.Global;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
-import org.elasticsearch.search.aggregations.bucket.missing.MissingBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.RangeBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.ipv4.IPv4RangeBuilder;
+import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.osgi.framework.BundleContext;
import org.osgi.framework.BundleEvent;
import org.osgi.framework.ServiceReference;
import org.osgi.framework.SynchronousBundleListener;
+import org.osgi.service.cm.ConfigurationAdmin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.management.*;
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXConnectorFactory;
+import javax.management.remote.JMXServiceURL;
import java.io.BufferedReader;
-import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.net.*;
-import java.nio.file.Paths;
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
+import java.lang.management.RuntimeMXBean;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.UnknownHostException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
-import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
-
@SuppressWarnings("rawtypes")
public class ElasticSearchPersistenceServiceImpl implements PersistenceService, ClusterService, SynchronousBundleListener {
private static final Logger logger = LoggerFactory.getLogger(ElasticSearchPersistenceServiceImpl.class.getName());
- public static final String DISCOVERY_ZEN_PING_MULTICAST_ENABLED = "discovery.zen.ping.multicast.enabled";
public static final String CONTEXTSERVER_ADDRESS = "contextserver.address";
public static final String CONTEXTSERVER_PORT = "contextserver.port";
public static final String CONTEXTSERVER_SECURE_ADDRESS = "contextserver.secureAddress";
public static final String CONTEXTSERVER_SECURE_PORT = "contextserver.securePort";
- public static final String KARAF_HOME = "karaf.home";
- public static final String ELASTICSEARCH_HOME_DIRECTORY = "elasticsearch";
- public static final String ELASTICSEARCH_PLUGINS_DIRECTORY = ELASTICSEARCH_HOME_DIRECTORY + "/plugins";
- public static final String ELASTICSEARCH_DATA_DIRECTORY = ELASTICSEARCH_HOME_DIRECTORY + "/data";
- public static final String INDEX_NUMBER_OF_REPLICAS = "index.number_of_replicas";
- public static final String INDEX_NUMBER_OF_SHARDS = "index.number_of_shards";
- public static final String NODE_CONTEXTSERVER_ADDRESS = "node.contextserver.address";
- public static final String NODE_CONTEXTSERVER_PORT = "node.contextserver.port";
- public static final String NODE_CONTEXTSERVER_SECURE_ADDRESS = "node.contextserver.secureAddress";
- public static final String NODE_CONTEXTSERVER_SECURE_PORT = "node.contextserver.securePort";
public static final String NUMBER_OF_SHARDS = "number_of_shards";
public static final String NUMBER_OF_REPLICAS = "number_of_replicas";
public static final String CLUSTER_NAME = "cluster.name";
- public static final String NODE_DATA = "node.data";
- public static final String PATH_DATA = "path.data";
- public static final String PATH_HOME = "path.home";
- public static final String PATH_PLUGINS = "path.plugins";
- public static final String INDEX_MAX_RESULT_WINDOW = "index.max_result_window";
- public static final String MAPPER_ALLOW_DOTS_IN_NAME = "mapper.allow_dots_in_name";
public static final String BULK_PROCESSOR_NAME = "bulkProcessor.name";
public static final String BULK_PROCESSOR_CONCURRENT_REQUESTS = "bulkProcessor.concurrentRequests";
public static final String BULK_PROCESSOR_BULK_ACTIONS = "bulkProcessor.bulkActions";
public static final String BULK_PROCESSOR_BULK_SIZE = "bulkProcessor.bulkSize";
public static final String BULK_PROCESSOR_FLUSH_INTERVAL = "bulkProcessor.flushInterval";
public static final String BULK_PROCESSOR_BACKOFF_POLICY = "bulkProcessor.backoffPolicy";
- public static final String ELASTICSEARCH_NETWORK_HOST = "network.host";
+ public static final String KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION = "org.apache.unoni.nodes";
+ public static final String KARAF_CLUSTER_CONFIGURATION_PUBLIC_ENDPOINTS = "publicEndpoints";
+ public static final String KARAF_CLUSTER_CONFIGURATION_SECURE_ENDPOINTS = "secureEndpoints";
- private Node node;
- private Client nodeClient;
private Client client;
private BulkProcessor bulkProcessor;
private String clusterName;
private String indexName;
private String monthlyIndexNumberOfShards;
private String monthlyIndexNumberOfReplicas;
- private String numberOfShards;
- private String numberOfReplicas;
- private Boolean nodeData;
- private Boolean discoveryEnabled;
private String elasticSearchConfig = null;
private BundleContext bundleContext;
private Map<String, String> mappings = new HashMap<String, String>();
private ConditionEvaluatorDispatcher conditionEvaluatorDispatcher;
private ConditionESQueryBuilderDispatcher conditionESQueryBuilderDispatcher;
+ private ClusterManager karafCellarClusterManager;
+ private EventProducer karafCellarEventProducer;
+ private GroupManager karafCellarGroupManager;
+ private String karafCellarGroupName = Configurations.DEFAULT_GROUP_NAME;
+ private ConfigurationAdmin osgiConfigurationAdmin;
+ private String karafJMXUsername = "karaf";
+ private String karafJMXPassword = "karaf";
+ private int karafJMXPort = 1099;
private Map<String,String> indexNames;
private List<String> itemsMonthlyIndexed;
@@ -203,22 +192,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
this.monthlyIndexNumberOfReplicas = monthlyIndexNumberOfReplicas;
}
- public void setDiscoveryEnabled(Boolean discoveryEnabled) {
- this.discoveryEnabled = discoveryEnabled;
- }
-
- public void setNumberOfShards(String numberOfShards) {
- this.numberOfShards = numberOfShards;
- }
-
- public void setNumberOfReplicas(String numberOfReplicas) {
- this.numberOfReplicas = numberOfReplicas;
- }
-
- public void setNodeData(Boolean nodeData) {
- this.nodeData = nodeData;
- }
-
public void setAddress(String address) {
this.address = address;
}
@@ -287,6 +260,38 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
this.bulkProcessorBackoffPolicy = bulkProcessorBackoffPolicy;
}
+ public void setKarafCellarClusterManager(ClusterManager karafCellarClusterManager) {
+ this.karafCellarClusterManager = karafCellarClusterManager;
+ }
+
+ public void setKarafCellarEventProducer(EventProducer karafCellarEventProducer) {
+ this.karafCellarEventProducer = karafCellarEventProducer;
+ }
+
+ public void setKarafCellarGroupManager(GroupManager karafCellarGroupManager) {
+ this.karafCellarGroupManager = karafCellarGroupManager;
+ }
+
+ public void setKarafCellarGroupName(String karafCellarGroupName) {
+ this.karafCellarGroupName = karafCellarGroupName;
+ }
+
+ public void setOsgiConfigurationAdmin(ConfigurationAdmin osgiConfigurationAdmin) {
+ this.osgiConfigurationAdmin = osgiConfigurationAdmin;
+ }
+
+ public void setKarafJMXUsername(String karafJMXUsername) {
+ this.karafJMXUsername = karafJMXUsername;
+ }
+
+ public void setKarafJMXPassword(String karafJMXPassword) {
+ this.karafJMXPassword = karafJMXPassword;
+ }
+
+ public void setKarafJMXPort(int karafJMXPort) {
+ this.karafJMXPort = karafJMXPort;
+ }
+
public void start() {
loadPredefinedMappings(bundleContext, false);
@@ -294,29 +299,58 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
// on startup
new InClassLoaderExecute<Object>() {
public Object execute(Object... args) {
- logger.info("Starting ElasticSearch persistence backend using cluster name " + clusterName + " and index name " + indexName + "...");
- Map<String, String> settings = null;
- if (elasticSearchConfig != null && elasticSearchConfig.length() > 0) {
- try {
- URI elasticSearchConfigURI = new URI(elasticSearchConfig);
- Settings.Builder settingsBuilder = Settings.builder().loadFromPath(Paths.get(elasticSearchConfigURI));
- settings = settingsBuilder.build().getAsMap();
- logger.info("Successfully loaded ElasticSearch configuration from " + elasticSearchConfigURI);
- } catch (URISyntaxException e) {
- logger.error("Error in ElasticSearch configuration URI ", e);
- } catch (SettingsException se) {
- logger.info("Error trying to load settings from " + elasticSearchConfig + ": " + se.getMessage() + " (activate debug mode for exception details)");
- if (logger.isDebugEnabled()) {
- logger.debug("Exception details", se);
- }
- }
- }
+ logger.info("Connecting to ElasticSearch persistence backend using cluster name " + clusterName + " and index name " + indexName + "...");
address = System.getProperty(CONTEXTSERVER_ADDRESS, address);
port = System.getProperty(CONTEXTSERVER_PORT, port);
secureAddress = System.getProperty(CONTEXTSERVER_SECURE_ADDRESS, secureAddress);
securePort = System.getProperty(CONTEXTSERVER_SECURE_PORT, securePort);
+ if (karafCellarEventProducer != null && karafCellarClusterManager != null) {
+
+ boolean setupConfigOk = true;
+ Group group = karafCellarGroupManager.findGroupByName(karafCellarGroupName);
+ if (setupConfigOk && group == null) {
+ logger.error("Cluster group " + karafCellarGroupName + " doesn't exist");
+ setupConfigOk = false;
+ }
+
+ // check if the producer is ON
+ if (setupConfigOk && karafCellarEventProducer.getSwitch().getStatus().equals(SwitchStatus.OFF)) {
+ logger.error("Cluster event producer is OFF");
+ setupConfigOk = false;
+ }
+
+ // check if the config pid is allowed
+ if (setupConfigOk && !isClusterConfigPIDAllowed(group, Constants.CATEGORY, KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION, EventType.OUTBOUND)) {
+ logger.error("Configuration PID " + KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION + " is blocked outbound for cluster group " + karafCellarGroupName);
+ setupConfigOk = false;
+ }
+
+ if (setupConfigOk) {
+ Map<String, Properties> configurations = karafCellarClusterManager.getMap(Constants.CONFIGURATION_MAP + Configurations.SEPARATOR + karafCellarGroupName);
+ org.apache.karaf.cellar.core.Node thisKarafNode = karafCellarClusterManager.getNode();
+ Properties karafCellarClusterNodeConfiguration = configurations.get(KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION);
+ if (karafCellarClusterNodeConfiguration == null) {
+ karafCellarClusterNodeConfiguration = new Properties();
+ }
+ String publicEndpointsPropValue = karafCellarClusterNodeConfiguration.getProperty(KARAF_CLUSTER_CONFIGURATION_PUBLIC_ENDPOINTS, thisKarafNode.getId() + "=" + address + ":" + port);
+ String secureEndpointsPropValue = karafCellarClusterNodeConfiguration.getProperty(KARAF_CLUSTER_CONFIGURATION_SECURE_ENDPOINTS, thisKarafNode.getId() + "=" + secureAddress + ":" + securePort);
+ String[] publicEndpointsArray = publicEndpointsPropValue.split(",");
+ Set<String> publicEndpoints = new TreeSet<String>(Arrays.asList(publicEndpointsArray));
+ String[] secureEndpointsArray = secureEndpointsPropValue.split(",");
+ Set<String> secureEndpoints = new TreeSet<String>(Arrays.asList(secureEndpointsArray));
+ publicEndpoints.add(thisKarafNode.getId() + "=" + address + ":" + port);
+ secureEndpoints.add(thisKarafNode.getId() + "=" + secureAddress + ":" + port);
+ karafCellarClusterNodeConfiguration.setProperty(KARAF_CLUSTER_CONFIGURATION_PUBLIC_ENDPOINTS, StringUtils.join(publicEndpoints, ","));
+ karafCellarClusterNodeConfiguration.setProperty(KARAF_CLUSTER_CONFIGURATION_SECURE_ENDPOINTS, StringUtils.join(secureEndpoints, ","));
+ configurations.put(KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION, karafCellarClusterNodeConfiguration);
+ ClusterConfigurationEvent clusterConfigurationEvent = new ClusterConfigurationEvent(KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION);
+ clusterConfigurationEvent.setSourceGroup(group);
+ karafCellarEventProducer.produce(clusterConfigurationEvent);
+ }
+ }
+
bulkProcessorName = System.getProperty(BULK_PROCESSOR_NAME, bulkProcessorName);
bulkProcessorConcurrentRequests = System.getProperty(BULK_PROCESSOR_CONCURRENT_REQUESTS, bulkProcessorConcurrentRequests);
bulkProcessorBulkActions = System.getProperty(BULK_PROCESSOR_BULK_ACTIONS, bulkProcessorBulkActions);
@@ -324,57 +358,13 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
bulkProcessorFlushInterval = System.getProperty(BULK_PROCESSOR_FLUSH_INTERVAL, bulkProcessorFlushInterval);
bulkProcessorBackoffPolicy = System.getProperty(BULK_PROCESSOR_BACKOFF_POLICY, bulkProcessorBackoffPolicy);
- Settings.Builder settingsBuilder = Settings.builder();
- if (settings != null) {
- settingsBuilder.put(settings);
- }
-
- String karafHome = System.getProperty(KARAF_HOME);
- File pluginsFile = new File(getConfig(settings, PATH_PLUGINS, new File(new File(karafHome), ELASTICSEARCH_PLUGINS_DIRECTORY).getAbsolutePath()));
- File homeFile = new File(getConfig(settings, PATH_HOME, new File(new File(karafHome), ELASTICSEARCH_HOME_DIRECTORY).getAbsolutePath()));
- File dataFile = new File(getConfig(settings, PATH_DATA, new File(new File(karafHome), ELASTICSEARCH_DATA_DIRECTORY).getAbsolutePath()));
-
- // allow dots in mappings (re-introduced in ElasticSearch 2.4.0)
- System.setProperty(MAPPER_ALLOW_DOTS_IN_NAME, "true");
-
- settingsBuilder.put(CLUSTER_NAME, clusterName)
- .put(NODE_DATA, nodeData)
- .put(PATH_DATA, dataFile.getAbsolutePath())
- .put(PATH_HOME, homeFile.getAbsolutePath())
- .put(PATH_PLUGINS, pluginsFile.getAbsolutePath())
- .put(DISCOVERY_ZEN_PING_MULTICAST_ENABLED, discoveryEnabled)
- .put(INDEX_NUMBER_OF_REPLICAS, numberOfReplicas)
- .put(INDEX_NUMBER_OF_SHARDS, numberOfShards)
- .put(NODE_CONTEXTSERVER_ADDRESS, address)
- .put(NODE_CONTEXTSERVER_PORT, port)
- .put(NODE_CONTEXTSERVER_SECURE_ADDRESS, secureAddress)
- .put(NODE_CONTEXTSERVER_SECURE_PORT, securePort)
- .put(INDEX_MAX_RESULT_WINDOW, "2147483647");
-
- if (settingsBuilder.get(ELASTICSEARCH_NETWORK_HOST) == null) {
- logger.info("Setting ElasticSearch network host address to {}", address);
- settingsBuilder.put(ELASTICSEARCH_NETWORK_HOST, address);
- }
-
- node = nodeBuilder().settings(settingsBuilder).node();
- nodeClient = node.client();
-
- logger.info("Waiting for ElasticSearch to start...");
-
- nodeClient.admin().cluster().prepareHealth()
- .setWaitForGreenStatus()
- .get();
-
- logger.info("Cluster status is GREEN");
-
try {
- Settings transportSettings = Settings.settingsBuilder()
+ Settings transportSettings = Settings.builder()
.put(CLUSTER_NAME, clusterName).build();
- client = TransportClient.builder().settings(transportSettings).build()
+ client = new PreBuiltTransportClient(transportSettings)
.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(address), 9300));
} catch (UnknownHostException e) {
logger.error("Error resolving address " + address + " ElasticSearch transport client not connected, using internal client instead", e);
- client = nodeClient;
}
// @todo is there a better way to detect index existence than to wait for it to startup ?
@@ -412,7 +402,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
client.admin().indices().preparePutTemplate(indexName + "_monthlyindex")
.setTemplate(indexName + "-*")
.setOrder(1)
- .setSettings(Settings.settingsBuilder()
+ .setSettings(Settings.builder()
.put(NUMBER_OF_SHARDS, Integer.parseInt(monthlyIndexNumberOfShards))
.put(NUMBER_OF_REPLICAS, Integer.parseInt(monthlyIndexNumberOfReplicas))
.build()).execute().actionGet();
@@ -582,11 +572,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
logger.error("Error waiting for bulk operations to flush !", e);
}
}
- if (nodeClient != client) {
+ if (client != null) {
client.close();
}
- nodeClient.close();
- node.close();
return null;
}
}.executeInClassLoader();
@@ -711,7 +699,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
String itemType = (String) clazz.getField("ITEM_TYPE").get(null);
if (itemsMonthlyIndexed.contains(itemType) && dateHint == null) {
- PartialList<T> r = query(QueryBuilders.idsQuery(itemType).ids(itemId), null, clazz, 0, 1, null);
+ PartialList<T> r = query(QueryBuilders.idsQuery(itemType).addIds(itemId), null, clazz, 0, 1, null);
if (r.size() > 0) {
return r.get(0);
}
@@ -886,11 +874,39 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
try {
String itemType = (String) clazz.getField("ITEM_TYPE").get(null);
- DeleteByQueryResponse rsp = new DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE)
+ BulkRequestBuilder deleteByScope = client.prepareBulk();
+
+ final TimeValue keepAlive = TimeValue.timeValueHours(1);
+ SearchResponse response = client.prepareSearch(indexName + "*")
.setIndices(getIndexNameForQuery(itemType))
+ .setScroll(keepAlive)
.setQuery(conditionESQueryBuilderDispatcher.getQueryBuilder(query))
- .execute()
- .actionGet();
+ .setSize(100).execute().actionGet();
+
+ // Scroll until no more hits are returned
+ while (true) {
+
+ for (SearchHit hit : response.getHits().getHits()) {
+ // add hit to bulk delete
+ deleteByScope.add(Requests.deleteRequest(hit.index()).type(hit.type()).id(hit.id()));
+ }
+
+ response = client.prepareSearchScroll(response.getScrollId()).setScroll(keepAlive).execute().actionGet();
+
+ // If we have no more hits, exit
+ if (response.getHits().getHits().length == 0) {
+ break;
+ }
+ }
+
+ // we're done with the scrolling, delete now
+ if (deleteByScope.numberOfActions() > 0) {
+ final BulkResponse deleteResponse = deleteByScope.get();
+ if (deleteResponse.hasFailures()) {
+ // do something
+ logger.debug("Couldn't remove by query " + query + ":\n{}", deleteResponse.buildFailureMessage());
+ }
+ }
return true;
} catch (Exception e) {
@@ -995,7 +1011,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = getMappingsResponse.getMappings();
Map<String, Map<String, Object>> propertyMap = new HashMap<>();
try {
- UnmodifiableIterator<ImmutableOpenMap<String, MappingMetaData>> it = mappings.valuesIt();
+ Iterator<ImmutableOpenMap<String, MappingMetaData>> it = mappings.valuesIt();
while (it.hasNext()) {
ImmutableOpenMap<String, MappingMetaData> next = it.next();
Map<String, Map<String, Object>> properties = (Map<String, Map<String, Object>>) next.get(itemType).getSourceAsMap().get("properties");
@@ -1030,7 +1046,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
logger.info("Saving query : " + queryName);
client.prepareIndex(indexName, ".percolator", queryName)
.setSource(query)
- .setRefresh(true) // Needed when the query shall be available immediately
+ .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
.execute().actionGet();
return true;
} catch (Exception e) {
@@ -1057,7 +1073,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
//Index the query = register it in the percolator
try {
client.prepareDelete(indexName, ".percolator", queryName)
- .setRefresh(true) // Needed when the query shall be available immediately
+ .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
.execute().actionGet();
return true;
} catch (Exception e) {
@@ -1069,36 +1085,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
@Override
- public List<String> getMatchingSavedQueries(final Item item) {
- return new InClassLoaderExecute<List<String>>() {
- protected List<String> execute(Object... args) {
- List<String> matchingQueries = new ArrayList<String>();
- try {
- String source = CustomObjectMapper.getObjectMapper().writeValueAsString(item);
-
- String itemType = item.getItemType();
-
- //Percolate
- PercolateResponse response = client.preparePercolate()
- .setIndices(indexName)
- .setDocumentType(itemType)
- .setSource("{doc:" + source + "}").execute().actionGet();
- //Iterate over the results
- for (PercolateResponse.Match match : response) {
- //Handle the result which is the name of
- //the query in the percolator
- matchingQueries.add(match.getId().string());
- }
- } catch (IOException e) {
- logger.error("Error getting matching saved queries for item=" + item, e);
- }
- return matchingQueries;
- }
- }.executeInClassLoader();
-
- }
-
- @Override
public boolean testMatch(Condition query, Item item) {
try {
return conditionEvaluatorDispatcher.eval(query, item);
@@ -1109,9 +1095,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
final Class<? extends Item> clazz = item.getClass();
String itemType = (String) clazz.getField("ITEM_TYPE").get(null);
- QueryBuilder builder = QueryBuilders.andQuery(
- QueryBuilders.idsQuery(itemType).ids(item.getItemId()),
- conditionESQueryBuilderDispatcher.buildFilter(query));
+ QueryBuilder builder = QueryBuilders.boolQuery()
+ .must(QueryBuilders.idsQuery(itemType).addIds(item.getItemId()))
+ .must(conditionESQueryBuilderDispatcher.buildFilter(query));
return queryCount(builder, itemType) > 0;
} catch (IllegalAccessException e) {
logger.error("Error getting query for item=" + item, e);
@@ -1181,9 +1167,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
protected Long execute(Object... args) {
SearchResponse response = client.prepareSearch(getIndexNameForQuery(itemType))
.setTypes(itemType)
- .setSearchType(SearchType.COUNT)
+ .setSize(0)
.setQuery(QueryBuilders.matchAllQuery())
- .addAggregation(AggregationBuilders.filter("filter").filter(filter))
+ .addAggregation(AggregationBuilders.filter("filter", filter))
.execute()
.actionGet();
Aggregations searchHits = response.getAggregations();
@@ -1213,7 +1199,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
} else if (size != -1) {
requestBuilder.setSize(size);
} else {
- requestBuilder.setSize(Integer.MAX_VALUE);
+ // requestBuilder.setSize(Integer.MAX_VALUE);
}
if (routing != null) {
requestBuilder.setRouting(routing);
@@ -1223,7 +1209,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
for (String sortByElement : sortByArray) {
if (sortByElement.startsWith("geo:")) {
String[] elements = sortByElement.split(":");
- GeoDistanceSortBuilder distanceSortBuilder = SortBuilders.geoDistanceSort(elements[1]).point(Double.parseDouble(elements[2]), Double.parseDouble(elements[3])).unit(DistanceUnit.KILOMETERS);
+ GeoDistanceSortBuilder distanceSortBuilder = SortBuilders.geoDistanceSort(elements[1], Double.parseDouble(elements[2]), Double.parseDouble(elements[3])).unit(DistanceUnit.KILOMETERS);
if (elements.length > 4 && elements[4].equals("desc")) {
requestBuilder = requestBuilder.addSort(distanceSortBuilder.order(SortOrder.DESC));
} else {
@@ -1270,7 +1256,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
SearchRequestBuilder builder = client.prepareSearch(getIndexNameForQuery(itemType))
.setTypes(itemType)
- .setSearchType(SearchType.COUNT)
+ .setSize(0)
.setQuery(QueryBuilders.matchAllQuery());
List<AggregationBuilder> lastAggregation = new ArrayList<AggregationBuilder>();
@@ -1279,13 +1265,13 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
AggregationBuilder bucketsAggregation = null;
if (aggregate instanceof DateAggregate) {
DateAggregate dateAggregate = (DateAggregate) aggregate;
- DateHistogramBuilder dateHistogramBuilder = AggregationBuilders.dateHistogram("buckets").field(aggregate.getField()).interval(new DateHistogramInterval((dateAggregate.getInterval())));
+ DateHistogramAggregationBuilder dateHistogramBuilder = AggregationBuilders.dateHistogram("buckets").field(aggregate.getField()).dateHistogramInterval(new DateHistogramInterval((dateAggregate.getInterval())));
if (dateAggregate.getFormat() != null) {
dateHistogramBuilder.format(dateAggregate.getFormat());
}
bucketsAggregation = dateHistogramBuilder;
} else if (aggregate instanceof NumericRangeAggregate) {
- RangeBuilder rangebuilder = AggregationBuilders.range("buckets").field(aggregate.getField());
+ RangeAggregationBuilder rangebuilder = AggregationBuilders.range("buckets").field(aggregate.getField());
for (NumericRange range : ((NumericRangeAggregate) aggregate).getRanges()) {
if (range != null) {
if (range.getFrom() != null && range.getTo() != null) {
@@ -1300,19 +1286,19 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
bucketsAggregation = rangebuilder;
} else if (aggregate instanceof DateRangeAggregate) {
DateRangeAggregate dateRangeAggregate = (DateRangeAggregate) aggregate;
- DateRangeBuilder rangebuilder = AggregationBuilders.dateRange("buckets").field(aggregate.getField());
+ DateRangeAggregationBuilder rangebuilder = AggregationBuilders.dateRange("buckets").field(aggregate.getField());
if (dateRangeAggregate.getFormat() != null) {
rangebuilder.format(dateRangeAggregate.getFormat());
}
for (DateRange range : dateRangeAggregate.getDateRanges()) {
if (range != null) {
- rangebuilder.addRange(range.getKey(), range.getFrom(), range.getTo());
+ rangebuilder.addRange(range.getKey(), range.getFrom().toString(), range.getTo().toString());
}
}
bucketsAggregation = rangebuilder;
} else if (aggregate instanceof IpRangeAggregate) {
IpRangeAggregate ipRangeAggregate = (IpRangeAggregate) aggregate;
- IPv4RangeBuilder rangebuilder = AggregationBuilders.ipRange("buckets").field(aggregate.getField());
+ IpRangeAggregationBuilder rangebuilder = AggregationBuilders.ipRange("buckets").field(aggregate.getField());
for (IpRange range : ipRangeAggregate.getRanges()) {
if (range != null) {
rangebuilder.addRange(range.getKey(), range.getFrom(), range.getTo());
@@ -1321,10 +1307,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
bucketsAggregation = rangebuilder;
} else {
//default
- bucketsAggregation = AggregationBuilders.terms("buckets").field(aggregate.getField()).size(Integer.MAX_VALUE);
+ bucketsAggregation = AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);
}
if (bucketsAggregation != null) {
- final MissingBuilder missingBucketsAggregation = AggregationBuilders.missing("missing").field(aggregate.getField());
+ final MissingAggregationBuilder missingBucketsAggregation = AggregationBuilders.missing("missing").field(aggregate.getField());
for (AggregationBuilder aggregationBuilder : lastAggregation) {
bucketsAggregation.subAggregation(aggregationBuilder);
missingBucketsAggregation.subAggregation(aggregationBuilder);
@@ -1334,7 +1320,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
if (filter != null) {
- AggregationBuilder filterAggregation = AggregationBuilders.filter("filter").filter(conditionESQueryBuilderDispatcher.buildFilter(filter));
+ AggregationBuilder filterAggregation = AggregationBuilders.filter("filter", conditionESQueryBuilderDispatcher.buildFilter(filter));
for (AggregationBuilder aggregationBuilder : lastAggregation) {
filterAggregation.subAggregation(aggregationBuilder);
}
@@ -1408,47 +1394,87 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
protected List<ClusterNode> execute(Object... args) {
Map<String, ClusterNode> clusterNodes = new LinkedHashMap<String, ClusterNode>();
- NodesInfoResponse nodesInfoResponse = client.admin().cluster().prepareNodesInfo(NodesInfoRequest.ALL_NODES)
- .setSettings(true)
- .execute()
- .actionGet();
- NodeInfo[] nodesInfoArray = nodesInfoResponse.getNodes();
- for (NodeInfo nodeInfo : nodesInfoArray) {
- if (nodeInfo.getSettings().get("node.contextserver.address") != null) {
- ClusterNode clusterNode = new ClusterNode();
- clusterNode.setHostName(nodeInfo.getHostname());
- clusterNode.setHostAddress(nodeInfo.getSettings().get("node.contextserver.address"));
- clusterNode.setPublicPort(Integer.parseInt(nodeInfo.getSettings().get("node.contextserver.port")));
- clusterNode.setSecureHostAddress(nodeInfo.getSettings().get("node.contextserver.secureAddress"));
- clusterNode.setSecurePort(Integer.parseInt(nodeInfo.getSettings().get("node.contextserver.securePort")));
- clusterNode.setMaster(nodeInfo.getNode().isMasterNode());
- clusterNode.setData(nodeInfo.getNode().isDataNode());
- clusterNodes.put(nodeInfo.getNode().getId(), clusterNode);
+ Set<org.apache.karaf.cellar.core.Node> karafCellarNodes = karafCellarClusterManager.listNodes();
+ org.apache.karaf.cellar.core.Node thisKarafNode = karafCellarClusterManager.getNode();
+ Map<String, Properties> clusterConfigurations = karafCellarClusterManager.getMap(Constants.CONFIGURATION_MAP + Configurations.SEPARATOR + karafCellarGroupName);
+ Properties karafCellarClusterNodeConfiguration = clusterConfigurations.get(KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION);
+ Map<String, String> publicNodeEndpoints = new TreeMap<>();
+ Map<String, String> secureNodeEndpoints = new TreeMap<>();
+ if (karafCellarClusterNodeConfiguration != null) {
+ String publicEndpointsPropValue = karafCellarClusterNodeConfiguration.getProperty(KARAF_CLUSTER_CONFIGURATION_PUBLIC_ENDPOINTS, thisKarafNode.getId() + "=" + address + ":" + port);
+ String secureEndpointsPropValue = karafCellarClusterNodeConfiguration.getProperty(KARAF_CLUSTER_CONFIGURATION_SECURE_ENDPOINTS, thisKarafNode.getId() + "=" + secureAddress + ":" + securePort);
+ String[] publicEndpointsArray = publicEndpointsPropValue.split(",");
+ Set<String> publicEndpoints = new TreeSet<String>(Arrays.asList(publicEndpointsArray));
+ for (String endpoint : publicEndpoints) {
+ String[] endpointParts = endpoint.split("=");
+ publicNodeEndpoints.put(endpointParts[0], endpointParts[1]);
+ }
+ String[] secureEndpointsArray = secureEndpointsPropValue.split(",");
+ Set<String> secureEndpoints = new TreeSet<String>(Arrays.asList(secureEndpointsArray));
+ for (String endpoint : secureEndpoints) {
+ String[] endpointParts = endpoint.split("=");
+ secureNodeEndpoints.put(endpointParts[0], endpointParts[1]);
}
}
-
- NodesStatsResponse nodesStatsResponse = client.admin().cluster().prepareNodesStats(NodesInfoRequest.ALL_NODES)
- .setOs(true)
- .setJvm(true)
- .setProcess(true)
- .execute()
- .actionGet();
- NodeStats[] nodeStatsArray = nodesStatsResponse.getNodes();
- for (NodeStats nodeStats : nodeStatsArray) {
- ClusterNode clusterNode = clusterNodes.get(nodeStats.getNode().getId());
- if (clusterNode != null) {
- // the following may be null in the case where Sigar didn't initialize properly, for example
- // because the native libraries were not installed or if we redeployed the OSGi bundle in which
- // case Sigar cannot initialize properly since it tries to reload the native libraries, generates
- // an error and doesn't initialize properly.
- if (nodeStats.getProcess() != null && nodeStats.getProcess().getCpu() != null) {
- clusterNode.setCpuLoad(nodeStats.getProcess().getCpu().getPercent());
+ for (org.apache.karaf.cellar.core.Node karafCellarNode : karafCellarNodes) {
+ ClusterNode clusterNode = new ClusterNode();
+ clusterNode.setHostName(karafCellarNode.getHost());
+ String publicEndpoint = publicNodeEndpoints.get(karafCellarNode.getId());
+ String[] publicEndpointParts = publicEndpoint.split(":");
+ clusterNode.setHostAddress(publicEndpointParts[0]);
+ clusterNode.setPublicPort(Integer.parseInt(publicEndpointParts[1]));
+ String secureEndpoint = secureNodeEndpoints.get(karafCellarNode.getId());
+ String[] secureEndpointParts = secureEndpoint.split(":");
+ clusterNode.setSecureHostAddress(secureEndpointParts[0]);
+ clusterNode.setSecurePort(Integer.parseInt(secureEndpointParts[1]));
+ clusterNode.setMaster(false);
+ clusterNode.setData(false);
+ try {
+ // now let's connect to remote JMX service to retrieve information from the runtime and operating system MX beans
+ JMXServiceURL url = new JMXServiceURL("service:jmx:rmi:///jndi/rmi://"+karafCellarNode.getHost() + ":"+karafJMXPort+"/karaf-root");
+ Map<String,Object> environment=new HashMap<String,Object>();
+ if (karafJMXUsername != null && karafJMXPassword != null) {
+ environment.put(JMXConnector.CREDENTIALS,new String[]{karafJMXUsername,karafJMXPassword});
}
- if (nodeStats.getOs() != null) {
- clusterNode.setLoadAverage(new double[] { nodeStats.getOs().getLoadAverage() });
- clusterNode.setUptime(nodeStats.getJvm().getUptime().getMillis());
+ JMXConnector jmxc = JMXConnectorFactory.connect(url, environment);
+ MBeanServerConnection mbsc = jmxc.getMBeanServerConnection();
+ final RuntimeMXBean remoteRuntime = ManagementFactory.newPlatformMXBeanProxy(mbsc, ManagementFactory.RUNTIME_MXBEAN_NAME, RuntimeMXBean.class);
+ clusterNode.setUptime(remoteRuntime.getUptime());
+ ObjectName operatingSystemMXBeanName = new ObjectName(ManagementFactory.OPERATING_SYSTEM_MXBEAN_NAME);
+ Double processCpuLoad = null;
+ Double systemCpuLoad = null;
+ try {
+ processCpuLoad = (Double) mbsc.getAttribute(operatingSystemMXBeanName, "ProcessCpuLoad");
+ } catch (MBeanException e) {
+ e.printStackTrace();
+ } catch (AttributeNotFoundException e) {
+ e.printStackTrace();
+ }
+ try {
+ systemCpuLoad = (Double) mbsc.getAttribute(operatingSystemMXBeanName, "SystemCpuLoad");
+ } catch (MBeanException e) {
+ e.printStackTrace();
+ } catch (AttributeNotFoundException e) {
+ e.printStackTrace();
}
+ final OperatingSystemMXBean remoteOperatingSystemMXBean = ManagementFactory.newPlatformMXBeanProxy(mbsc, ManagementFactory.OPERATING_SYSTEM_MXBEAN_NAME, OperatingSystemMXBean.class);
+ clusterNode.setLoadAverage(new double[] { remoteOperatingSystemMXBean.getSystemLoadAverage()});
+ if (systemCpuLoad != null) {
+ clusterNode.setCpuLoad(systemCpuLoad);
+ }
+
+ } catch (MalformedURLException e) {
+ logger.error("Error connecting to remote JMX server", e);
+ } catch (IOException e) {
+ logger.error("Error retrieving remote JMX data", e);
+ } catch (MalformedObjectNameException e) {
+ logger.error("Error retrieving remote JMX data", e);
+ } catch (InstanceNotFoundException e) {
+ logger.error("Error retrieving remote JMX data", e);
+ } catch (ReflectionException e) {
+ logger.error("Error retrieving remote JMX data", e);
}
+ clusterNodes.put(karafCellarNode.getId(), clusterNode);
}
return new ArrayList<ClusterNode>(clusterNodes.values());
@@ -1486,7 +1512,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
.setFlush(false)
.setCompletion(false)
.setRefresh(false)
- .setSuggest(false)
.execute()
.actionGet();
@@ -1525,7 +1550,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
final TimeValue keepAlive = TimeValue.timeValueHours(1);
SearchResponse response = client.prepareSearch(indexName + "*")
- .setSearchType(SearchType.SCAN)
.setScroll(keepAlive)
.setQuery(query)
.setSize(100).execute().actionGet();
@@ -1570,9 +1594,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
SearchRequestBuilder builder = client.prepareSearch(getIndexNameForQuery(itemType))
.setTypes(itemType)
- .setSearchType(SearchType.COUNT)
+ .setSize(0)
.setQuery(QueryBuilders.matchAllQuery());
- AggregationBuilder filterAggregation = AggregationBuilders.filter("metrics").filter(conditionESQueryBuilderDispatcher.buildFilter(condition));
+ AggregationBuilder filterAggregation = AggregationBuilders.filter("metrics", conditionESQueryBuilderDispatcher.buildFilter(condition));
if (metrics != null) {
for (String metric : metrics) {
@@ -1638,4 +1662,22 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
return defaultValue;
}
+
+ /**
+ * Check if a configuration is allowed.
+ *
+ * @param group the cluster group.
+ * @param category the configuration category constant.
+ * @param pid the configuration PID.
+ * @param type the cluster event type.
+ * @return true if the cluster event type is allowed, false else.
+ */
+ public boolean isClusterConfigPIDAllowed(Group group, String category, String pid, EventType type) {
+ CellarSupport support = new CellarSupport();
+ support.setClusterManager(this.karafCellarClusterManager);
+ support.setGroupManager(this.karafCellarGroupManager);
+ support.setConfigurationAdmin(this.osgiConfigurationAdmin);
+ return support.isAllowed(group, category, pid, type);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/actionType.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/actionType.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/actionType.json
index 9470350..fa2f0ea 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/actionType.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/actionType.json
@@ -4,9 +4,9 @@
{
"all": {
"match": "*",
- "match_mapping_type": "string",
+ "match_mapping_type": "text",
"mapping": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
index 36788d7..efdcb3c 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
@@ -4,9 +4,9 @@
{
"all": {
"match": "*",
- "match_mapping_type": "string",
+ "match_mapping_type": "text",
"mapping": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
}
}
@@ -17,8 +17,7 @@
"type": "double"
},
"currency": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"startDate": {
"type": "date",
@@ -29,25 +28,21 @@
"format": "strict_date_optional_time||epoch_millis"
},
"itemId": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"itemType": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"primaryGoal": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"timezone": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"metadata": {
"properties": {
"description": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
},
"enabled": {
@@ -57,25 +52,22 @@
"type": "boolean"
},
"id": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"missingPlugins": {
"type": "boolean"
},
"name": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"readOnly": {
"type": "boolean"
},
"scope": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"tags": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
index ccc001e..f682e6a 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
@@ -4,9 +4,9 @@
{
"all": {
"match": "*",
- "match_mapping_type": "string",
+ "match_mapping_type": "text",
"mapping": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
}
}
@@ -14,36 +14,31 @@
],
"properties": {
"campaignId": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"cost": {
"type": "double"
},
"currency": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"eventDate": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"itemId": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"itemType": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"timezone": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"metadata": {
"properties": {
"description": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
},
"enabled": {
@@ -53,22 +48,19 @@
"type": "boolean"
},
"id": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"missingPlugins": {
"type": "boolean"
},
"name": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
},
"readOnly": {
"type": "boolean"
},
"scope": {
- "type": "string",
- "analyzer": "folding"
+ "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/96a018ec/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/conditionType.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/conditionType.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/conditionType.json
index 73ecac9..702a9f1 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/conditionType.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/conditionType.json
@@ -4,13 +4,73 @@
{
"all": {
"match": "*",
- "match_mapping_type": "string",
+ "match_mapping_type": "text",
"mapping": {
- "type": "string",
+ "type": "text",
"analyzer": "folding"
}
}
}
- ]
+ ],
+ "properties" : {
+ "conditionEvaluator" : {
+ "type" : "keyword"
+ },
+ "itemId" : {
+ "type" : "keyword"
+ },
+ "itemType" : {
+ "type" : "keyword"
+ },
+ "metadata" : {
+ "properties" : {
+ "description" : {
+ "type" : "text",
+ "analyzer" : "folding"
+ },
+ "enabled" : {
+ "type" : "boolean"
+ },
+ "hidden" : {
+ "type" : "boolean"
+ },
+ "id" : {
+ "type" : "keyword"
+ },
+ "missingPlugins" : {
+ "type" : "boolean"
+ },
+ "name" : {
+ "type" : "keyword"
+ },
+ "readOnly" : {
+ "type" : "boolean"
+ },
+ "tags" : {
+ "type" : "text",
+ "analyzer" : "folding"
+ }
+ }
+ },
+ "parameters" : {
+ "properties" : {
+ "defaultValue" : {
+ "type" : "keyword"
+ },
+ "id" : {
+ "type" : "keyword"
+ },
+ "multivalued" : {
+ "type" : "boolean"
+ },
+ "type" : {
+ "type" : "keyword"
+ }
+ }
+ },
+ "queryBuilder" : {
+ "type" : "keyword"
+ }
+ }
}
}
\ No newline at end of file
[20/50] [abbrv] incubator-unomi git commit: UNOMI-70 Upgrade to
ElasticSearch 5.x - Use scroll API to get unlimited results
Posted by sh...@apache.org.
UNOMI-70 Upgrade to ElasticSearch 5.x
- Use scroll API to get unlimited results
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/a94fdb0e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/a94fdb0e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/a94fdb0e
Branch: refs/heads/master
Commit: a94fdb0e02b3606c012fa5f3d6d35212a34b489d
Parents: dafa9f6
Author: Serge Huber <sh...@apache.org>
Authored: Thu Dec 8 09:14:14 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Thu Dec 8 09:14:14 2016 +0100
----------------------------------------------------------------------
.../ElasticSearchPersistenceServiceImpl.java | 47 ++++++++++++++++----
1 file changed, 39 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a94fdb0e/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 0dc3899..fa4b5e1 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -898,6 +898,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
break;
}
}
+ client.prepareClearScroll().addScrollId(response.getScrollId()).execute().actionGet();
// we're done with the scrolling, delete now
if (deleteByScope.numberOfActions() > 0) {
@@ -1188,6 +1189,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
long totalHits = 0;
try {
String itemType = getItemType(clazz);
+ final TimeValue keepAlive = TimeValue.timeValueHours(1);
SearchRequestBuilder requestBuilder = client.prepareSearch(getIndexNameForQuery(itemType))
.setTypes(itemType)
@@ -1199,7 +1201,14 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
} else if (size != -1) {
requestBuilder.setSize(size);
} else {
- // requestBuilder.setSize(Integer.MAX_VALUE);
+ // size == -1, use scroll query to retrieve all the results
+ requestBuilder = client.prepareSearch(getIndexNameForQuery(itemType))
+ .setTypes(itemType)
+ .setFetchSource(true)
+ .setScroll(keepAlive)
+ .setFrom(offset)
+ .setQuery(query)
+ .setSize(100);
}
if (routing != null) {
requestBuilder.setRouting(routing);
@@ -1229,13 +1238,35 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
SearchResponse response = requestBuilder
.execute()
.actionGet();
- SearchHits searchHits = response.getHits();
- totalHits = searchHits.getTotalHits();
- for (SearchHit searchHit : searchHits) {
- String sourceAsString = searchHit.getSourceAsString();
- final T value = CustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz);
- value.setItemId(searchHit.getId());
- results.add(value);
+ if (size == -1) {
+ // Scroll until no more hits are returned
+ while (true) {
+
+ for (SearchHit searchHit : response.getHits().getHits()) {
+ // add hit to results
+ String sourceAsString = searchHit.getSourceAsString();
+ final T value = CustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz);
+ value.setItemId(searchHit.getId());
+ results.add(value);
+ }
+
+ response = client.prepareSearchScroll(response.getScrollId()).setScroll(keepAlive).execute().actionGet();
+
+ // If we have no more hits, exit
+ if (response.getHits().getHits().length == 0) {
+ break;
+ }
+ }
+ client.prepareClearScroll().addScrollId(response.getScrollId()).execute().actionGet();
+ } else {
+ SearchHits searchHits = response.getHits();
+ totalHits = searchHits.getTotalHits();
+ for (SearchHit searchHit : searchHits) {
+ String sourceAsString = searchHit.getSourceAsString();
+ final T value = CustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz);
+ value.setItemId(searchHit.getId());
+ results.add(value);
+ }
}
} catch (Exception t) {
logger.error("Error loading itemType=" + clazz.getName() + " query=" + query + " sortBy=" + sortBy, t);
[19/50] [abbrv] incubator-unomi git commit: UNOMI-70 Upgrade to
ElasticSearch 5.x - Working on getting integration testing working (not yet
working)
Posted by sh...@apache.org.
UNOMI-70 Upgrade to ElasticSearch 5.x
- Working on getting integration testing working (not yet working)
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/dafa9f6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/dafa9f6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/dafa9f6b
Branch: refs/heads/master
Commit: dafa9f6b3c68a37084b4148ce7c76c7faf5687e1
Parents: a217ba0
Author: Serge Huber <sh...@apache.org>
Authored: Tue Dec 6 13:48:48 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Tue Dec 6 13:48:48 2016 +0100
----------------------------------------------------------------------
itests/pom.xml | 11 +-
.../java/org/apache/unomi/itests/AllITs.java | 38 ++++
.../java/org/apache/unomi/itests/AllTests.java | 38 ----
.../java/org/apache/unomi/itests/BaseIT.java | 108 +++++++++++
.../java/org/apache/unomi/itests/BaseTest.java | 100 -----------
.../java/org/apache/unomi/itests/BasicIT.java | 99 +++++++++++
.../java/org/apache/unomi/itests/BasicTest.java | 99 -----------
.../unomi/itests/ConditionESQueryBuilderIT.java | 59 ++++++
.../itests/ConditionESQueryBuilderTest.java | 59 ------
.../unomi/itests/ConditionEvaluatorIT.java | 178 +++++++++++++++++++
.../unomi/itests/ConditionEvaluatorTest.java | 178 -------------------
.../apache/unomi/itests/ProfileServiceIT.java | 53 ++++++
.../apache/unomi/itests/ProfileServiceTest.java | 53 ------
.../java/org/apache/unomi/itests/SegmentIT.java | 48 +++++
.../org/apache/unomi/itests/SegmentTest.java | 48 -----
pom.xml | 7 +-
16 files changed, 598 insertions(+), 578 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/dafa9f6b/itests/pom.xml
----------------------------------------------------------------------
diff --git a/itests/pom.xml b/itests/pom.xml
index 11e0a94..0bde83c 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -54,6 +54,12 @@
<type>xml</type>
</dependency>
<dependency>
+ <groupId>org.apache.karaf.cellar</groupId>
+ <artifactId>apache-karaf-cellar</artifactId>
+ <classifier>features</classifier>
+ <type>xml</type>
+ </dependency>
+ <dependency>
<groupId>org.ops4j.pax.web</groupId>
<artifactId>pax-web-features</artifactId>
<classifier>features</classifier>
@@ -140,8 +146,9 @@
<version>5.0</version>
<configuration>
<clusterName>contextElasticSearch</clusterName>
- <tcpPort>9300</tcpPort>
+ <transportPort>9300</transportPort>
<httpPort>9200</httpPort>
+ <version>5.0.2</version>
</configuration>
<executions>
<!--
@@ -169,7 +176,7 @@
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<includes>
- <include>**/*AllTests.java</include>
+ <include>**/*AllITs.java</include>
</includes>
</configuration>
<executions>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/dafa9f6b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/AllITs.java b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
new file mode 100644
index 0000000..96e5245
--- /dev/null
+++ b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package org.apache.unomi.itests;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+import org.junit.runners.Suite.SuiteClasses;
+
+/**
+ * Defines suite of test classes to run.
+ *
+ * @author Sergiy Shyrkov
+ */
+@RunWith(Suite.class)
+@SuiteClasses({
+ //BasicIT.class,
+ ConditionEvaluatorIT.class,
+ ConditionESQueryBuilderIT.class,
+ SegmentIT.class,
+ ProfileServiceIT.class
+ })
+public class AllITs {
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/dafa9f6b/itests/src/test/java/org/apache/unomi/itests/AllTests.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/AllTests.java b/itests/src/test/java/org/apache/unomi/itests/AllTests.java
deleted file mode 100644
index 0fbd383..0000000
--- a/itests/src/test/java/org/apache/unomi/itests/AllTests.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-
-package org.apache.unomi.itests;
-
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-import org.junit.runners.Suite.SuiteClasses;
-
-/**
- * Defines suite of test classes to run.
- *
- * @author Sergiy Shyrkov
- */
-@RunWith(Suite.class)
-@SuiteClasses({
- //BasicTest.class,
- ConditionEvaluatorTest.class,
- ConditionESQueryBuilderTest.class,
- SegmentTest.class,
- ProfileServiceTest.class
- })
-public class AllTests {
-}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/dafa9f6b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java
new file mode 100644
index 0000000..334e7c6
--- /dev/null
+++ b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package org.apache.unomi.itests;
+
+import org.ops4j.pax.exam.Configuration;
+import org.ops4j.pax.exam.Option;
+import org.ops4j.pax.exam.karaf.options.LogLevelOption.LogLevel;
+import org.ops4j.pax.exam.options.MavenArtifactUrlReference;
+import org.ops4j.pax.exam.options.MavenUrlReference;
+
+import java.io.File;
+
+import static org.ops4j.pax.exam.CoreOptions.*;
+import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.*;
+
+/**
+ * Base class for integration tests.
+ *
+ * @author kevan
+ */
+public abstract class BaseIT {
+
+ protected static final String HTTP_PORT = "8181";
+
+ protected static final String URL = "http://localhost:" + HTTP_PORT;
+
+ @Configuration
+ public Option[] config() {
+ MavenArtifactUrlReference karafUrl = maven()
+ .groupId("org.apache.karaf")
+ .artifactId("apache-karaf")
+ .version("3.0.8")
+ .type("tar.gz");
+
+ MavenUrlReference karafStandardRepo = maven()
+ .groupId("org.apache.karaf.features")
+ .artifactId("standard")
+ .classifier("features")
+ .type("xml")
+ .versionAsInProject();
+ MavenUrlReference karafCellarRepo = maven()
+ .groupId("org.apache.karaf.cellar")
+ .artifactId("apache-karaf-cellar")
+ .classifier("features")
+ .type("xml")
+ .versionAsInProject();
+ MavenUrlReference karafPaxWebRepo = maven()
+ .groupId("org.ops4j.pax.web")
+ .artifactId("pax-web-features")
+ .classifier("features")
+ .type("xml")
+ .versionAsInProject();
+ MavenUrlReference karafCxfRepo = maven()
+ .groupId("org.apache.cxf.karaf")
+ .artifactId("apache-cxf")
+ .classifier("features")
+ .type("xml")
+ .versionAsInProject();
+ MavenUrlReference contextServerRepo = maven()
+ .groupId("org.apache.unomi")
+ .artifactId("unomi-kar")
+ .classifier("features")
+ .type("xml")
+ .versionAsInProject();
+
+ return new Option[]{
+ debugConfiguration("5005", false),
+ karafDistributionConfiguration()
+ .frameworkUrl(karafUrl)
+ .unpackDirectory(new File("target/exam"))
+ .useDeployFolder(false),
+ keepRuntimeFolder(),
+ configureConsole().ignoreLocalConsole().ignoreRemoteShell(),
+ logLevel(LogLevel.INFO),
+// editConfigurationFilePut("etc/org.ops4j.pax.web.cfg", "org.osgi.service.http.port", HTTP_PORT),
+// systemProperty("org.osgi.service.http.port").value(HTTP_PORT),
+ systemProperty("org.ops4j.pax.exam.rbc.rmi.port").value("1199"),
+ features(karafPaxWebRepo, "war"),
+ features(karafCxfRepo, "cxf"),
+ features(karafStandardRepo, "openwebbeans"),
+ features(karafStandardRepo, "pax-cdi-web-openwebbeans"),
+ features(karafCellarRepo, "cellar"),
+ features(contextServerRepo, "unomi-kar"),
+ // we need to wrap the HttpComponents libraries ourselves since the OSGi bundles provided by the project are incorrect
+ wrappedBundle(mavenBundle("org.apache.httpcomponents",
+ "httpcore").versionAsInProject()),
+ wrappedBundle(mavenBundle("org.apache.httpcomponents",
+ "httpmime").versionAsInProject()),
+ wrappedBundle(mavenBundle("org.apache.httpcomponents",
+ "httpclient").versionAsInProject())
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/dafa9f6b/itests/src/test/java/org/apache/unomi/itests/BaseTest.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/BaseTest.java b/itests/src/test/java/org/apache/unomi/itests/BaseTest.java
deleted file mode 100644
index 9db3d9b..0000000
--- a/itests/src/test/java/org/apache/unomi/itests/BaseTest.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-
-package org.apache.unomi.itests;
-
-import org.ops4j.pax.exam.Configuration;
-import org.ops4j.pax.exam.Option;
-import org.ops4j.pax.exam.karaf.options.LogLevelOption.LogLevel;
-import org.ops4j.pax.exam.options.MavenArtifactUrlReference;
-import org.ops4j.pax.exam.options.MavenUrlReference;
-
-import java.io.File;
-
-import static org.ops4j.pax.exam.CoreOptions.*;
-import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.*;
-
-/**
- * Base class for integration tests.
- *
- * @author kevan
- */
-public abstract class BaseTest {
-
- protected static final String HTTP_PORT = "8181";
-
- protected static final String URL = "http://localhost:" + HTTP_PORT;
-
- @Configuration
- public Option[] config() {
- MavenArtifactUrlReference karafUrl = maven()
- .groupId("org.apache.karaf")
- .artifactId("apache-karaf")
- .version("3.0.2")
- .type("tar.gz");
-
- MavenUrlReference karafStandardRepo = maven()
- .groupId("org.apache.karaf.features")
- .artifactId("standard")
- .classifier("features")
- .type("xml")
- .versionAsInProject();
- MavenUrlReference karafPaxWebRepo = maven()
- .groupId("org.ops4j.pax.web")
- .artifactId("pax-web-features")
- .classifier("features")
- .type("xml")
- .versionAsInProject();
- MavenUrlReference karafCxfRepo = maven()
- .groupId("org.apache.cxf.karaf")
- .artifactId("apache-cxf")
- .classifier("features")
- .type("xml")
- .versionAsInProject();
- MavenUrlReference contextServerRepo = maven()
- .groupId("org.apache.unomi")
- .artifactId("unomi-kar")
- .classifier("features")
- .type("xml")
- .versionAsInProject();
-
- return new Option[]{
- debugConfiguration("5005", false),
- karafDistributionConfiguration()
- .frameworkUrl(karafUrl)
- .unpackDirectory(new File("target/exam"))
- .useDeployFolder(false),
-// keepRuntimeFolder(),
- configureConsole().ignoreLocalConsole().ignoreRemoteShell(),
- logLevel(LogLevel.INFO),
-// editConfigurationFilePut("etc/org.ops4j.pax.web.cfg", "org.osgi.service.http.port", HTTP_PORT),
-// systemProperty("org.osgi.service.http.port").value(HTTP_PORT),
- features(karafPaxWebRepo, "war"),
- features(karafCxfRepo, "cxf"),
- features(karafStandardRepo, "openwebbeans"),
- features(karafStandardRepo, "pax-cdi-web-openwebbeans"),
- features(contextServerRepo, "unomi-kar"),
- // we need to wrap the HttpComponents libraries ourselves since the OSGi bundles provided by the project are incorrect
- wrappedBundle(mavenBundle("org.apache.httpcomponents",
- "httpcore").versionAsInProject()),
- wrappedBundle(mavenBundle("org.apache.httpcomponents",
- "httpmime").versionAsInProject()),
- wrappedBundle(mavenBundle("org.apache.httpcomponents",
- "httpclient").versionAsInProject())
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/dafa9f6b/itests/src/test/java/org/apache/unomi/itests/BasicIT.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/BasicIT.java b/itests/src/test/java/org/apache/unomi/itests/BasicIT.java
new file mode 100644
index 0000000..cc91fed
--- /dev/null
+++ b/itests/src/test/java/org/apache/unomi/itests/BasicIT.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package org.apache.unomi.itests;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.util.EntityUtils;
+import org.apache.unomi.api.ContextRequest;
+import org.apache.unomi.api.ContextResponse;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.ops4j.pax.exam.junit.PaxExam;
+import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy;
+import org.ops4j.pax.exam.spi.reactors.PerSuite;
+
+import java.io.IOException;
+
+@RunWith(PaxExam.class)
+@ExamReactorStrategy(PerSuite.class)
+public class BasicIT extends BaseIT {
+ private static final String JSON_MYME_TYPE = "application/json";
+
+ private ObjectMapper objectMapper = new ObjectMapper();
+
+ @Test
+ public void testContextJS() throws IOException {
+ HttpUriRequest request = new HttpGet(URL + "/context.js?sessionId=aa3b04bd-8f4d-4a07-8e96-d33ffa04d3d9");
+ CloseableHttpResponse response = HttpClientBuilder.create().build().execute(request);
+ // The underlying HTTP connection is still held by the response object
+ // to allow the response content to be streamed directly from the network socket.
+ // In order to ensure correct deallocation of system resources
+ // the profile MUST call CloseableHttpResponse#close() from a finally clause.
+ // Please note that if response content is not fully consumed the underlying
+ // connection cannot be safely re-used and will be shut down and discarded
+ // by the connection manager.
+ String responseContent = null;
+ try {
+ System.out.println(response.getStatusLine());
+ HttpEntity entity = response.getEntity();
+ // do something useful with the response body
+ // and ensure it is fully consumed
+ responseContent = EntityUtils.toString(entity);
+ } finally {
+ response.close();
+ }
+ Assert.assertTrue("Response should contain context object", responseContent.contains("window.digitalData = window.digitalData || {};"));
+ // @todo we should check the validity of the context object, but this is rather complex since it would
+ // potentially require parsing the Javascript !
+ }
+
+ @Test
+ public void testContextJSON() throws IOException {
+ String sessionId = "aa3b04bd-8f4d-4a07-8e96-d33ffa04d3d9";
+ ContextRequest contextRequest = new ContextRequest();
+// contextRequest.setSource(new EventSource());
+// contextRequest.getSource().setId("af6f393a-a537-4586-991b-8521b9c7b05b");
+ HttpPost request = new HttpPost(URL + "/context.json?sessionId=" + sessionId);
+ request.setEntity(new StringEntity(objectMapper.writeValueAsString(contextRequest), ContentType.create("application/json")));
+ CloseableHttpResponse response = HttpClientBuilder.create().build().execute(request);
+
+ try {
+ // validate mimeType
+ String mimeType = ContentType.getOrDefault(response.getEntity()).getMimeType();
+ Assert.assertEquals("Response content type should be " + JSON_MYME_TYPE, JSON_MYME_TYPE, mimeType);
+
+ // validate context
+ ContextResponse context = TestUtils.retrieveResourceFromResponse(response, ContextResponse.class);
+ Assert.assertNotNull("Context should not be null", context);
+ Assert.assertNotNull("Context profileId should not be null", context.getProfileId());
+ Assert.assertEquals("Context sessionId should be the same as the sessionId used to request the context", sessionId, context.getSessionId());
+ } finally {
+ response.close();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/dafa9f6b/itests/src/test/java/org/apache/unomi/itests/BasicTest.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/BasicTest.java b/itests/src/test/java/org/apache/unomi/itests/BasicTest.java
deleted file mode 100644
index 7bfba87..0000000
--- a/itests/src/test/java/org/apache/unomi/itests/BasicTest.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-
-package org.apache.unomi.itests;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.util.EntityUtils;
-import org.apache.unomi.api.ContextRequest;
-import org.apache.unomi.api.ContextResponse;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.ops4j.pax.exam.junit.PaxExam;
-import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy;
-import org.ops4j.pax.exam.spi.reactors.PerSuite;
-
-import java.io.IOException;
-
-@RunWith(PaxExam.class)
-@ExamReactorStrategy(PerSuite.class)
-public class BasicTest extends BaseTest{
- private static final String JSON_MYME_TYPE = "application/json";
-
- private ObjectMapper objectMapper = new ObjectMapper();
-
- @Test
- public void testContextJS() throws IOException {
- HttpUriRequest request = new HttpGet(URL + "/context.js?sessionId=aa3b04bd-8f4d-4a07-8e96-d33ffa04d3d9");
- CloseableHttpResponse response = HttpClientBuilder.create().build().execute(request);
- // The underlying HTTP connection is still held by the response object
- // to allow the response content to be streamed directly from the network socket.
- // In order to ensure correct deallocation of system resources
- // the profile MUST call CloseableHttpResponse#close() from a finally clause.
- // Please note that if response content is not fully consumed the underlying
- // connection cannot be safely re-used and will be shut down and discarded
- // by the connection manager.
- String responseContent = null;
- try {
- System.out.println(response.getStatusLine());
- HttpEntity entity = response.getEntity();
- // do something useful with the response body
- // and ensure it is fully consumed
- responseContent = EntityUtils.toString(entity);
- } finally {
- response.close();
- }
- Assert.assertTrue("Response should contain context object", responseContent.contains("window.digitalData = window.digitalData || {};"));
- // @todo we should check the validity of the context object, but this is rather complex since it would
- // potentially require parsing the Javascript !
- }
-
- @Test
- public void testContextJSON() throws IOException {
- String sessionId = "aa3b04bd-8f4d-4a07-8e96-d33ffa04d3d9";
- ContextRequest contextRequest = new ContextRequest();
-// contextRequest.setSource(new EventSource());
-// contextRequest.getSource().setId("af6f393a-a537-4586-991b-8521b9c7b05b");
- HttpPost request = new HttpPost(URL + "/context.json?sessionId=" + sessionId);
- request.setEntity(new StringEntity(objectMapper.writeValueAsString(contextRequest), ContentType.create("application/json")));
- CloseableHttpResponse response = HttpClientBuilder.create().build().execute(request);
-
- try {
- // validate mimeType
- String mimeType = ContentType.getOrDefault(response.getEntity()).getMimeType();
- Assert.assertEquals("Response content type should be " + JSON_MYME_TYPE, JSON_MYME_TYPE, mimeType);
-
- // validate context
- ContextResponse context = TestUtils.retrieveResourceFromResponse(response, ContextResponse.class);
- Assert.assertNotNull("Context should not be null", context);
- Assert.assertNotNull("Context profileId should not be null", context.getProfileId());
- Assert.assertEquals("Context sessionId should be the same as the sessionId used to request the context", sessionId, context.getSessionId());
- } finally {
- response.close();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/dafa9f6b/itests/src/test/java/org/apache/unomi/itests/ConditionESQueryBuilderIT.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/ConditionESQueryBuilderIT.java b/itests/src/test/java/org/apache/unomi/itests/ConditionESQueryBuilderIT.java
new file mode 100644
index 0000000..3e4333c
--- /dev/null
+++ b/itests/src/test/java/org/apache/unomi/itests/ConditionESQueryBuilderIT.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package org.apache.unomi.itests;
+
+import org.apache.unomi.api.Item;
+import org.apache.unomi.api.conditions.Condition;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.ops4j.pax.exam.junit.PaxExam;
+import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy;
+import org.ops4j.pax.exam.spi.reactors.PerSuite;
+
+import java.util.List;
+
+/**
+ * Integration tests for various condition query builder types (elasticsearch).
+ *
+ * @author Sergiy Shyrkov
+ */
+@RunWith(PaxExam.class)
+@ExamReactorStrategy(PerSuite.class)
+public class ConditionESQueryBuilderIT extends ConditionEvaluatorIT {
+
+ @Override
+ protected boolean eval(Condition c) {
+ @SuppressWarnings("unchecked")
+ List<Item> list = persistenceService.query(c,null,(Class<Item>) item.getClass());
+ return list.contains(item);
+ }
+
+ @Before
+ public void setUp() {
+ super.setUp();
+ persistenceService.save(item);
+ persistenceService.refresh();
+ }
+
+ @After
+ public void tearDown() {
+ persistenceService.remove(item.getItemId(), item.getClass());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/dafa9f6b/itests/src/test/java/org/apache/unomi/itests/ConditionESQueryBuilderTest.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/ConditionESQueryBuilderTest.java b/itests/src/test/java/org/apache/unomi/itests/ConditionESQueryBuilderTest.java
deleted file mode 100644
index 82e1780..0000000
--- a/itests/src/test/java/org/apache/unomi/itests/ConditionESQueryBuilderTest.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-
-package org.apache.unomi.itests;
-
-import org.apache.unomi.api.Item;
-import org.apache.unomi.api.conditions.Condition;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.runner.RunWith;
-import org.ops4j.pax.exam.junit.PaxExam;
-import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy;
-import org.ops4j.pax.exam.spi.reactors.PerSuite;
-
-import java.util.List;
-
-/**
- * Integration tests for various condition query builder types (elasticsearch).
- *
- * @author Sergiy Shyrkov
- */
-@RunWith(PaxExam.class)
-@ExamReactorStrategy(PerSuite.class)
-public class ConditionESQueryBuilderTest extends ConditionEvaluatorTest {
-
- @Override
- protected boolean eval(Condition c) {
- @SuppressWarnings("unchecked")
- List<Item> list = persistenceService.query(c,null,(Class<Item>) item.getClass());
- return list.contains(item);
- }
-
- @Before
- public void setUp() {
- super.setUp();
- persistenceService.save(item);
- persistenceService.refresh();
- }
-
- @After
- public void tearDown() {
- persistenceService.remove(item.getItemId(), item.getClass());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/dafa9f6b/itests/src/test/java/org/apache/unomi/itests/ConditionEvaluatorIT.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/ConditionEvaluatorIT.java b/itests/src/test/java/org/apache/unomi/itests/ConditionEvaluatorIT.java
new file mode 100644
index 0000000..b75a8f5
--- /dev/null
+++ b/itests/src/test/java/org/apache/unomi/itests/ConditionEvaluatorIT.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package org.apache.unomi.itests;
+
+import org.apache.unomi.api.Item;
+import org.apache.unomi.api.Profile;
+import org.apache.unomi.api.conditions.Condition;
+import org.apache.unomi.api.services.DefinitionsService;
+import org.apache.unomi.persistence.spi.PersistenceService;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.ops4j.pax.exam.junit.PaxExam;
+import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy;
+import org.ops4j.pax.exam.spi.reactors.PerSuite;
+
+import javax.inject.Inject;
+import java.util.*;
+
+import static org.junit.Assert.*;
+
+/**
+ * Integration tests for various condition types.
+ *
+ * @author Sergiy Shyrkov
+ */
+@RunWith(PaxExam.class)
+@ExamReactorStrategy(PerSuite.class)
+public class ConditionEvaluatorIT extends BaseIT {
+
+ protected ConditionBuilder builder;
+ @Inject
+ protected PersistenceService persistenceService;
+ protected Item item;
+ protected Date lastVisit;
+ @Inject
+ private DefinitionsService definitionsService;
+
+ protected boolean eval(Condition c) {
+ return persistenceService.testMatch(c, item);
+ }
+
+ @Before
+ public void setUp() {
+ assertNotNull("Definition service should be available", definitionsService);
+ assertNotNull("Persistence service should be available", persistenceService);
+
+ lastVisit = new GregorianCalendar(2015,1,1,20,30,0).getTime();
+
+ Profile profile = new Profile("profile-" + UUID.randomUUID().toString());
+ profile.setProperty("firstVisit", lastVisit);
+ profile.setProperty("age", Integer.valueOf(30));
+ profile.setProperty("gender", "female");
+ profile.setProperty("lastVisit", lastVisit);
+ profile.setSegments(new HashSet<String>(Arrays.asList("s1", "s2", "s3")));
+ this.item = profile;
+ builder = new ConditionBuilder(definitionsService);
+ }
+
+ @Test
+ public void testCompound() {
+ // test AND
+ assertTrue(eval(builder.and(builder.profileProperty("properties.gender").equalTo("female"),
+ builder.profileProperty("properties.age").equalTo(Integer.valueOf(30))).build()));
+ assertFalse(eval(builder.and(builder.profileProperty("properties.gender").equalTo("male"),
+ builder.profileProperty("properties.age").equalTo(Integer.valueOf(30))).build()));
+ assertFalse(eval(builder.and(builder.profileProperty("properties.gender").equalTo("female"),
+ builder.profileProperty("properties.age").equalTo(Integer.valueOf(40))).build()));
+
+ // test OR
+ assertTrue(eval(builder.or(builder.profileProperty("properties.gender").equalTo("female"),
+ builder.profileProperty("properties.age").equalTo(Integer.valueOf(40))).build()));
+ assertTrue(eval(builder.or(builder.profileProperty("properties.gender").equalTo("male"),
+ builder.profileProperty("properties.age").equalTo(Integer.valueOf(30))).build()));
+ assertFalse(eval(builder.or(builder.profileProperty("properties.gender").equalTo("male"),
+ builder.profileProperty("properties.age").equalTo(Integer.valueOf(40))).build()));
+
+ // test NOT
+ assertTrue(eval(builder.not(builder.profileProperty("properties.gender").equalTo("male")).build()));
+ assertFalse(eval(builder.not(builder.profileProperty("properties.age").equalTo(Integer.valueOf(30))).build()));
+
+ }
+
+ @Test
+ public void testDate() {
+ assertTrue(eval(builder.profileProperty("properties.lastVisit").equalTo(lastVisit).build()));
+ assertTrue(eval(builder.profileProperty("properties.lastVisit")
+ .greaterThan(new Date(lastVisit.getTime() - 10000)).build()));
+ assertTrue(eval(builder.profileProperty("properties.lastVisit").lessThan(new Date(lastVisit.getTime() + 10000))
+ .build()));
+ assertTrue(eval(builder.profileProperty("properties.lastVisit")
+ .in(new Date(lastVisit.getTime() + 10000), new Date(lastVisit.getTime() - 10000), lastVisit).build()));
+ assertTrue(eval(builder.profileProperty("properties.lastVisit")
+ .notIn(new Date(lastVisit.getTime() + 10000), new Date(lastVisit.getTime() - 10000)).build()));
+ assertFalse(eval(builder.profileProperty("properties.lastVisit")
+ .notIn(new Date(lastVisit.getTime() + 10000), new Date(lastVisit.getTime() - 10000), lastVisit).build()));
+ assertTrue(eval(builder.profileProperty("properties.lastVisit").all(lastVisit).build()));
+ assertFalse(eval(builder.profileProperty("properties.lastVisit")
+ .all(new Date(lastVisit.getTime() + 10000), lastVisit).build()));
+ }
+
+ @Test
+ public void testExistence() {
+ assertTrue("Gender property does not exist",
+ eval(builder.profileProperty("properties.gender").exists().build()));
+ assertFalse("Gender property missing", eval(builder.profileProperty("properties.gender").missing().build()));
+ assertTrue("Strange property exists", eval(builder.profileProperty("properties.unknown").missing().build()));
+ assertFalse("Strange property exists", eval(builder.profileProperty("properties.unknown").exists().build()));
+ }
+
+ @Test
+ public void testInteger() {
+ assertTrue(eval(builder.profileProperty("properties.age").equalTo(Integer.valueOf(30)).build()));
+ assertTrue(eval(builder.not(builder.profileProperty("properties.age").equalTo(Integer.valueOf(40))).build()));
+ assertTrue(eval(builder.profileProperty("properties.age").notEqualTo(Integer.valueOf(40)).build()));
+ assertTrue(eval(builder.profileProperty("properties.age").lessThan(Integer.valueOf(40)).build()));
+ assertTrue(eval(builder.profileProperty("properties.age").greaterThan(Integer.valueOf(20)).build()));
+ assertTrue(eval(builder.profileProperty("properties.age").greaterThanOrEqualTo(Integer.valueOf(30)).build()));
+ assertFalse(eval(builder.profileProperty("properties.age").greaterThanOrEqualTo(Integer.valueOf(31)).build()));
+
+ assertTrue(eval(builder.profileProperty("properties.age").in(Integer.valueOf(30)).build()));
+ assertTrue(eval(builder.profileProperty("properties.age").in(Integer.valueOf(31), Integer.valueOf(30)).build()));
+ assertTrue(eval(builder.profileProperty("properties.age").notIn(Integer.valueOf(25), Integer.valueOf(26))
+ .build()));
+ assertFalse(eval(builder.profileProperty("properties.age").notIn(Integer.valueOf(25), Integer.valueOf(30))
+ .build()));
+ }
+
+ @Test
+ public void testMultiValue() {
+ assertTrue(eval(builder.property("profileSegmentCondition", "segments").parameter("matchType", "in")
+ .parameter("segments", "s10", "s20", "s2").build()));
+ assertFalse(eval(builder.property("profileSegmentCondition", "segments").parameter("matchType", "in")
+ .parameter("segments", "s10", "s20", "s30").build()));
+ assertTrue(eval(builder.property("profileSegmentCondition", "segments").parameter("matchType", "notIn")
+ .parameter("segments", "s10", "s20", "s30").build()));
+ assertFalse(eval(builder.property("profileSegmentCondition", "segments").parameter("matchType", "notIn")
+ .parameter("segments", "s10", "s20", "s2").build()));
+ assertTrue(eval(builder.property("profileSegmentCondition", "segments").parameter("matchType", "all")
+ .parameter("segments", "s1", "s2").build()));
+ assertFalse(eval(builder.property("profileSegmentCondition", "segments").parameter("matchType", "all")
+ .parameter("segments", "s1", "s5").build()));
+ }
+
+ @Test
+ public void testString() {
+ assertTrue(eval(builder.profileProperty("properties.gender").equalTo("female").build()));
+ assertFalse(eval(builder.not(builder.profileProperty("properties.gender").equalTo("female")).build()));
+ assertTrue(eval(builder.profileProperty("properties.gender").notEqualTo("male").build()));
+// assertFalse(eval(builder.not(builder.profileProperty("properties.gender").notEqualTo("male")).build()));
+ assertTrue(eval(builder.profileProperty("properties.gender").startsWith("fe").build()));
+ assertTrue(eval(builder.profileProperty("properties.gender").endsWith("le").build()));
+ assertTrue(eval(builder.profileProperty("properties.gender").contains("fem").build()));
+ assertFalse(eval(builder.profileProperty("properties.gender").contains("mu").build()));
+ assertTrue(eval(builder.profileProperty("properties.gender").matchesRegex(".*ale").build()));
+
+ assertTrue(eval(builder.profileProperty("properties.gender").in("male", "female").build()));
+ assertTrue(eval(builder.profileProperty("properties.gender").notIn("one", "two").build()));
+ assertFalse(eval(builder.profileProperty("properties.gender").notIn("one", "two", "female").build()));
+ assertTrue(eval(builder.profileProperty("properties.gender").all("female").build()));
+ //assertFalse(eval(builder.profileProperty("properties.gender").all("male", "female").build()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/dafa9f6b/itests/src/test/java/org/apache/unomi/itests/ConditionEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/ConditionEvaluatorTest.java b/itests/src/test/java/org/apache/unomi/itests/ConditionEvaluatorTest.java
deleted file mode 100644
index c635daf..0000000
--- a/itests/src/test/java/org/apache/unomi/itests/ConditionEvaluatorTest.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-
-package org.apache.unomi.itests;
-
-import org.apache.unomi.api.Item;
-import org.apache.unomi.api.Profile;
-import org.apache.unomi.api.conditions.Condition;
-import org.apache.unomi.api.services.DefinitionsService;
-import org.apache.unomi.persistence.spi.PersistenceService;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.ops4j.pax.exam.junit.PaxExam;
-import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy;
-import org.ops4j.pax.exam.spi.reactors.PerSuite;
-
-import javax.inject.Inject;
-import java.util.*;
-
-import static org.junit.Assert.*;
-
-/**
- * Integration tests for various condition types.
- *
- * @author Sergiy Shyrkov
- */
-@RunWith(PaxExam.class)
-@ExamReactorStrategy(PerSuite.class)
-public class ConditionEvaluatorTest extends BaseTest {
-
- protected ConditionBuilder builder;
- @Inject
- protected PersistenceService persistenceService;
- protected Item item;
- protected Date lastVisit;
- @Inject
- private DefinitionsService definitionsService;
-
- protected boolean eval(Condition c) {
- return persistenceService.testMatch(c, item);
- }
-
- @Before
- public void setUp() {
- assertNotNull("Definition service should be available", definitionsService);
- assertNotNull("Persistence service should be available", persistenceService);
-
- lastVisit = new GregorianCalendar(2015,1,1,20,30,0).getTime();
-
- Profile profile = new Profile("profile-" + UUID.randomUUID().toString());
- profile.setProperty("firstVisit", lastVisit);
- profile.setProperty("age", Integer.valueOf(30));
- profile.setProperty("gender", "female");
- profile.setProperty("lastVisit", lastVisit);
- profile.setSegments(new HashSet<String>(Arrays.asList("s1", "s2", "s3")));
- this.item = profile;
- builder = new ConditionBuilder(definitionsService);
- }
-
- @Test
- public void testCompound() {
- // test AND
- assertTrue(eval(builder.and(builder.profileProperty("properties.gender").equalTo("female"),
- builder.profileProperty("properties.age").equalTo(Integer.valueOf(30))).build()));
- assertFalse(eval(builder.and(builder.profileProperty("properties.gender").equalTo("male"),
- builder.profileProperty("properties.age").equalTo(Integer.valueOf(30))).build()));
- assertFalse(eval(builder.and(builder.profileProperty("properties.gender").equalTo("female"),
- builder.profileProperty("properties.age").equalTo(Integer.valueOf(40))).build()));
-
- // test OR
- assertTrue(eval(builder.or(builder.profileProperty("properties.gender").equalTo("female"),
- builder.profileProperty("properties.age").equalTo(Integer.valueOf(40))).build()));
- assertTrue(eval(builder.or(builder.profileProperty("properties.gender").equalTo("male"),
- builder.profileProperty("properties.age").equalTo(Integer.valueOf(30))).build()));
- assertFalse(eval(builder.or(builder.profileProperty("properties.gender").equalTo("male"),
- builder.profileProperty("properties.age").equalTo(Integer.valueOf(40))).build()));
-
- // test NOT
- assertTrue(eval(builder.not(builder.profileProperty("properties.gender").equalTo("male")).build()));
- assertFalse(eval(builder.not(builder.profileProperty("properties.age").equalTo(Integer.valueOf(30))).build()));
-
- }
-
- @Test
- public void testDate() {
- assertTrue(eval(builder.profileProperty("properties.lastVisit").equalTo(lastVisit).build()));
- assertTrue(eval(builder.profileProperty("properties.lastVisit")
- .greaterThan(new Date(lastVisit.getTime() - 10000)).build()));
- assertTrue(eval(builder.profileProperty("properties.lastVisit").lessThan(new Date(lastVisit.getTime() + 10000))
- .build()));
- assertTrue(eval(builder.profileProperty("properties.lastVisit")
- .in(new Date(lastVisit.getTime() + 10000), new Date(lastVisit.getTime() - 10000), lastVisit).build()));
- assertTrue(eval(builder.profileProperty("properties.lastVisit")
- .notIn(new Date(lastVisit.getTime() + 10000), new Date(lastVisit.getTime() - 10000)).build()));
- assertFalse(eval(builder.profileProperty("properties.lastVisit")
- .notIn(new Date(lastVisit.getTime() + 10000), new Date(lastVisit.getTime() - 10000), lastVisit).build()));
- assertTrue(eval(builder.profileProperty("properties.lastVisit").all(lastVisit).build()));
- assertFalse(eval(builder.profileProperty("properties.lastVisit")
- .all(new Date(lastVisit.getTime() + 10000), lastVisit).build()));
- }
-
- @Test
- public void testExistence() {
- assertTrue("Gender property does not exist",
- eval(builder.profileProperty("properties.gender").exists().build()));
- assertFalse("Gender property missing", eval(builder.profileProperty("properties.gender").missing().build()));
- assertTrue("Strange property exists", eval(builder.profileProperty("properties.unknown").missing().build()));
- assertFalse("Strange property exists", eval(builder.profileProperty("properties.unknown").exists().build()));
- }
-
- @Test
- public void testInteger() {
- assertTrue(eval(builder.profileProperty("properties.age").equalTo(Integer.valueOf(30)).build()));
- assertTrue(eval(builder.not(builder.profileProperty("properties.age").equalTo(Integer.valueOf(40))).build()));
- assertTrue(eval(builder.profileProperty("properties.age").notEqualTo(Integer.valueOf(40)).build()));
- assertTrue(eval(builder.profileProperty("properties.age").lessThan(Integer.valueOf(40)).build()));
- assertTrue(eval(builder.profileProperty("properties.age").greaterThan(Integer.valueOf(20)).build()));
- assertTrue(eval(builder.profileProperty("properties.age").greaterThanOrEqualTo(Integer.valueOf(30)).build()));
- assertFalse(eval(builder.profileProperty("properties.age").greaterThanOrEqualTo(Integer.valueOf(31)).build()));
-
- assertTrue(eval(builder.profileProperty("properties.age").in(Integer.valueOf(30)).build()));
- assertTrue(eval(builder.profileProperty("properties.age").in(Integer.valueOf(31), Integer.valueOf(30)).build()));
- assertTrue(eval(builder.profileProperty("properties.age").notIn(Integer.valueOf(25), Integer.valueOf(26))
- .build()));
- assertFalse(eval(builder.profileProperty("properties.age").notIn(Integer.valueOf(25), Integer.valueOf(30))
- .build()));
- }
-
- @Test
- public void testMultiValue() {
- assertTrue(eval(builder.property("profileSegmentCondition", "segments").parameter("matchType", "in")
- .parameter("segments", "s10", "s20", "s2").build()));
- assertFalse(eval(builder.property("profileSegmentCondition", "segments").parameter("matchType", "in")
- .parameter("segments", "s10", "s20", "s30").build()));
- assertTrue(eval(builder.property("profileSegmentCondition", "segments").parameter("matchType", "notIn")
- .parameter("segments", "s10", "s20", "s30").build()));
- assertFalse(eval(builder.property("profileSegmentCondition", "segments").parameter("matchType", "notIn")
- .parameter("segments", "s10", "s20", "s2").build()));
- assertTrue(eval(builder.property("profileSegmentCondition", "segments").parameter("matchType", "all")
- .parameter("segments", "s1", "s2").build()));
- assertFalse(eval(builder.property("profileSegmentCondition", "segments").parameter("matchType", "all")
- .parameter("segments", "s1", "s5").build()));
- }
-
- @Test
- public void testString() {
- assertTrue(eval(builder.profileProperty("properties.gender").equalTo("female").build()));
- assertFalse(eval(builder.not(builder.profileProperty("properties.gender").equalTo("female")).build()));
- assertTrue(eval(builder.profileProperty("properties.gender").notEqualTo("male").build()));
-// assertFalse(eval(builder.not(builder.profileProperty("properties.gender").notEqualTo("male")).build()));
- assertTrue(eval(builder.profileProperty("properties.gender").startsWith("fe").build()));
- assertTrue(eval(builder.profileProperty("properties.gender").endsWith("le").build()));
- assertTrue(eval(builder.profileProperty("properties.gender").contains("fem").build()));
- assertFalse(eval(builder.profileProperty("properties.gender").contains("mu").build()));
- assertTrue(eval(builder.profileProperty("properties.gender").matchesRegex(".*ale").build()));
-
- assertTrue(eval(builder.profileProperty("properties.gender").in("male", "female").build()));
- assertTrue(eval(builder.profileProperty("properties.gender").notIn("one", "two").build()));
- assertFalse(eval(builder.profileProperty("properties.gender").notIn("one", "two", "female").build()));
- assertTrue(eval(builder.profileProperty("properties.gender").all("female").build()));
- //assertFalse(eval(builder.profileProperty("properties.gender").all("male", "female").build()));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/dafa9f6b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
new file mode 100644
index 0000000..c23e5ea
--- /dev/null
+++ b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+package org.apache.unomi.itests;
+
+import org.apache.unomi.api.Profile;
+import org.apache.unomi.api.services.ProfileService;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.ops4j.pax.exam.junit.PaxExam;
+import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy;
+import org.ops4j.pax.exam.spi.reactors.PerSuite;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+
+/**
+ * An integration test for the profile service
+ */
+@RunWith(PaxExam.class)
+@ExamReactorStrategy(PerSuite.class)
+public class ProfileServiceIT extends BaseIT {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(SegmentIT.class);
+ private final static String TEST_PROFILE_ID = "test-profile-id";
+ @Inject
+ protected ProfileService profileService;
+
+ @Test
+ public void testProfileDelete() {
+ Profile profile = new Profile();
+ profile.setItemId(TEST_PROFILE_ID);
+ profileService.save(profile);
+ LOGGER.info("Profile saved, now testing profile delete...");
+ profileService.delete(TEST_PROFILE_ID, false);
+ LOGGER.info("Profile deleted successfully.");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/dafa9f6b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceTest.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileServiceTest.java b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceTest.java
deleted file mode 100644
index 4f5dba8..0000000
--- a/itests/src/test/java/org/apache/unomi/itests/ProfileServiceTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-package org.apache.unomi.itests;
-
-import org.apache.unomi.api.Profile;
-import org.apache.unomi.api.services.ProfileService;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.ops4j.pax.exam.junit.PaxExam;
-import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy;
-import org.ops4j.pax.exam.spi.reactors.PerSuite;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-
-/**
- * An integration test for the profile service
- */
-@RunWith(PaxExam.class)
-@ExamReactorStrategy(PerSuite.class)
-public class ProfileServiceTest extends BaseTest {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(SegmentTest.class);
- private final static String TEST_PROFILE_ID = "test-profile-id";
- @Inject
- protected ProfileService profileService;
-
- @Test
- public void testProfileDelete() {
- Profile profile = new Profile();
- profile.setItemId(TEST_PROFILE_ID);
- profileService.save(profile);
- LOGGER.info("Profile saved, now testing profile delete...");
- profileService.delete(TEST_PROFILE_ID, false);
- LOGGER.info("Profile deleted successfully.");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/dafa9f6b/itests/src/test/java/org/apache/unomi/itests/SegmentIT.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/SegmentIT.java b/itests/src/test/java/org/apache/unomi/itests/SegmentIT.java
new file mode 100644
index 0000000..721ab8f
--- /dev/null
+++ b/itests/src/test/java/org/apache/unomi/itests/SegmentIT.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package org.apache.unomi.itests;
+
+import org.apache.unomi.api.Metadata;
+import org.apache.unomi.api.services.SegmentService;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.ops4j.pax.exam.junit.PaxExam;
+import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy;
+import org.ops4j.pax.exam.spi.reactors.PerSuite;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.util.List;
+
+@RunWith(PaxExam.class)
+@ExamReactorStrategy(PerSuite.class)
+public class SegmentIT extends BaseIT {
+ private final static Logger LOGGER = LoggerFactory.getLogger(SegmentIT.class);
+ @Inject
+ protected SegmentService segmentService;
+
+ @Test
+ public void testSegments() {
+ Assert.assertNotNull("Segment service should be available", segmentService);
+ List<Metadata> segmentMetadatas = segmentService.getSegmentMetadatas(0, 50, null).getList();
+ Assert.assertEquals("Segment metadata list should be empty", 0, segmentMetadatas.size());
+ LOGGER.info("Retrieved " + segmentMetadatas.size() + " segment metadata entries");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/dafa9f6b/itests/src/test/java/org/apache/unomi/itests/SegmentTest.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/SegmentTest.java b/itests/src/test/java/org/apache/unomi/itests/SegmentTest.java
deleted file mode 100644
index 8bf5169..0000000
--- a/itests/src/test/java/org/apache/unomi/itests/SegmentTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-
-package org.apache.unomi.itests;
-
-import org.apache.unomi.api.Metadata;
-import org.apache.unomi.api.services.SegmentService;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.ops4j.pax.exam.junit.PaxExam;
-import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy;
-import org.ops4j.pax.exam.spi.reactors.PerSuite;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import java.util.List;
-
-@RunWith(PaxExam.class)
-@ExamReactorStrategy(PerSuite.class)
-public class SegmentTest extends BaseTest{
- private final static Logger LOGGER = LoggerFactory.getLogger(SegmentTest.class);
- @Inject
- protected SegmentService segmentService;
-
- @Test
- public void testSegments() {
- Assert.assertNotNull("Segment service should be available", segmentService);
- List<Metadata> segmentMetadatas = segmentService.getSegmentMetadatas(0, 50, null).getList();
- Assert.assertEquals("Segment metadata list should be empty", 0, segmentMetadatas.size());
- LOGGER.info("Retrieved " + segmentMetadatas.size() + " segment metadata entries");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/dafa9f6b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1d8b334..040e382 100644
--- a/pom.xml
+++ b/pom.xml
@@ -904,7 +904,12 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
- <version>2.18.1</version>
+ <version>2.19.1</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>2.19.1</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
[28/50] [abbrv] incubator-unomi git commit: UNOMI-70 : Avoid npe when
mapping is not found
Posted by sh...@apache.org.
UNOMI-70 : Avoid npe when mapping is not found
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/69b0403d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/69b0403d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/69b0403d
Branch: refs/heads/master
Commit: 69b0403d7763d650bcc1e5883eec7e2d6a4d65f0
Parents: 8b5a44b
Author: Thomas Draier <dr...@apache.org>
Authored: Tue Dec 20 15:29:50 2016 +0100
Committer: Thomas Draier <dr...@apache.org>
Committed: Tue Dec 20 15:29:50 2016 +0100
----------------------------------------------------------------------
.../ElasticSearchPersistenceServiceImpl.java | 16 ++++++----------
1 file changed, 6 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/69b0403d/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index ee85d98..49e09d0 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -1054,20 +1054,16 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private Map<String, Object> getPropertyMapping(String property, Map<String, Map<String, Object>> mappings) {
Map<String, Object> propMapping = null;
String[] properties = StringUtils.split(property, '.');
- for (int i = 0; i < properties.length; i++) {
+ for (int i = 0; i < properties.length && mappings != null; i++) {
String s = properties[i];
- if (mappings != null) {
- propMapping = mappings.get(s);
- if (i == properties.length - 1) {
- return propMapping;
- } else {
- mappings = (Map<String, Map<String, Object>>) propMapping.get("properties");
- }
+ propMapping = mappings.get(s);
+ if (i == properties.length - 1) {
+ return propMapping;
} else {
- return null;
+ mappings = propMapping != null ? ((Map<String, Map<String, Object>>) propMapping.get("properties")) : null;
}
}
- return propMapping;
+ return null;
}
public boolean saveQuery(final String queryName, final String query) {
[30/50] [abbrv] incubator-unomi git commit: - Introduce a new save
method that supports batching - Use new save method batching to import
Geonames (a lot) faster - Replace slow running ASCIIFoldingFilter call with
faster MappingCharFilter call. See https
Posted by sh...@apache.org.
- Introduce a new save method that supports batching
- Use new save method batching to import Geonames (a lot) faster
- Replace slow running ASCIIFoldingFilter call with faster MappingCharFilter call. See https://issues.apache.org/jira/browse/LUCENE-7525 and https://issues.apache.org/jira/browse/SOLR-2013
-
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/ac2c9ba5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/ac2c9ba5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/ac2c9ba5
Branch: refs/heads/master
Commit: ac2c9ba5c6b955048ac47afb402dc80e1213f16f
Parents: 69b0403
Author: Serge Huber <sh...@apache.org>
Authored: Tue Dec 20 16:38:19 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Tue Dec 20 16:38:46 2016 +0100
----------------------------------------------------------------------
.../geonames/services/GeonamesServiceImpl.java | 26 +-
.../ElasticSearchPersistenceServiceImpl.java | 11 +-
.../conditions/ConditionContextHelper.java | 31 +-
.../src/main/resources/mapping-FoldToASCII.txt | 3813 ++++++++++++++++++
.../persistence/spi/PersistenceService.java | 11 +
5 files changed, 3879 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ac2c9ba5/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java
----------------------------------------------------------------------
diff --git a/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java b/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java
index ebe53a0..2b0a0dc 100644
--- a/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java
+++ b/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java
@@ -112,12 +112,15 @@ public class GeonamesServiceImpl implements GeonamesService {
ZipInputStream zipInputStream = new ZipInputStream(new FileInputStream(f));
ZipEntry zipEntry = zipInputStream.getNextEntry(); // used to advance to the first entry in the ZipInputStream
+ long fileSize = zipEntry.getSize();
BufferedReader reader = new BufferedReader(new InputStreamReader(zipInputStream, "UTF-8"));
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
String line;
logger.info("Starting to import geonames database from file {}...", f);
- long lineCount = 0;
+ long charCount = 0;
+ double lastCompletionPourcentage = 0.0;
+ long lastCharCount = 0;
long importStartTime = System.currentTimeMillis();
while ((line = reader.readLine()) != null) {
String[] values = line.split("\t");
@@ -134,14 +137,25 @@ public class GeonamesServiceImpl implements GeonamesService {
values[16], values[17],
sdf.parse(values[18]));
- persistenceService.save(geonameEntry);
+ persistenceService.save(geonameEntry, true);
}
- lineCount++;
- if (lineCount % 1000 == 0) {
- logger.info("{} lines imported from file {}", lineCount, f);
+ charCount+=line.length();
+ if (fileSize > 0) {
+ double completionPourcentage = 100.0 * charCount / fileSize;
+ if (completionPourcentage - lastCompletionPourcentage > 1.0) {
+ int roundedPourcentage = (int) completionPourcentage;
+ logger.info("{}% imported from file {}", roundedPourcentage, f);
+ lastCompletionPourcentage = completionPourcentage;
+ }
+ } else {
+ if (charCount - lastCharCount > (100*1024*1024)) {
+ logger.info("{}MB imported from file {}", charCount / (1024*1024), f);
+ lastCharCount = charCount;
+ }
}
}
- logger.info("{} lines from Geonames database file {} imported in {}ms", lineCount, f, System.currentTimeMillis()-importStartTime);
+ long totalTimeMillis = System.currentTimeMillis()-importStartTime;
+ logger.info("{} characters from Geonames database file {} imported in {}ms. Speed={}MB/s", charCount, f, totalTimeMillis, charCount / (1024*1024) / (totalTimeMillis / 1000));
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ac2c9ba5/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 49e09d0..e7ed75b 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -739,6 +739,11 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
@Override
public boolean save(final Item item) {
+ return save(item, false);
+ }
+
+ @Override
+ public boolean save(final Item item, final boolean useBatching) {
return new InClassLoaderExecute<Boolean>() {
protected Boolean execute(Object... args) {
@@ -769,7 +774,11 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
try {
- indexBuilder.execute().actionGet();
+ if (bulkProcessor == null || !useBatching) {
+ indexBuilder.execute().actionGet();
+ } else {
+ bulkProcessor.add(indexBuilder.request());
+ }
} catch (IndexNotFoundException e) {
if (existingIndexNames.contains(index)) {
existingIndexNames.remove(index);
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ac2c9ba5/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java
index 63964e1..6f3f1b4 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java
@@ -20,14 +20,18 @@ package org.apache.unomi.persistence.elasticsearch.conditions;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
-import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
-import org.apache.lucene.util.ArrayUtil;
+import org.apache.logging.log4j.core.util.IOUtils;
+import org.apache.lucene.analysis.charfilter.MappingCharFilterFactory;
+import org.apache.lucene.analysis.util.ClasspathResourceLoader;
import org.apache.unomi.api.conditions.Condition;
import org.mvel2.MVEL;
import org.mvel2.ParserConfiguration;
import org.mvel2.ParserContext;
+import java.io.IOException;
+import java.io.Reader;
import java.io.Serializable;
+import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -37,6 +41,18 @@ import java.util.concurrent.ConcurrentHashMap;
public class ConditionContextHelper {
private static Map<String,Serializable> mvelExpressions = new ConcurrentHashMap<>();
+ private static MappingCharFilterFactory mappingCharFilterFactory;
+ static {
+ Map<String,String> args = new HashMap<>();
+ args.put("mapping", "mapping-FoldToASCII.txt");
+ mappingCharFilterFactory = new MappingCharFilterFactory(args);
+ try {
+ mappingCharFilterFactory.inform(new ClasspathResourceLoader(ConditionContextHelper.class.getClassLoader()));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
public static Condition getContextualCondition(Condition condition, Map<String, Object> context) {
if (context.isEmpty() || !hasContextualParameter(condition.getParameterValues())) {
return condition;
@@ -124,10 +140,13 @@ public class ConditionContextHelper {
public static String foldToASCII(String s) {
if (s != null) {
s = s.toLowerCase();
- int maxSizeNeeded = 4 * s.length();
- char[] output = new char[ArrayUtil.oversize(maxSizeNeeded, 2)];
- int length = ASCIIFoldingFilter.foldToASCII(s.toCharArray(), 0, output, 0, s.length());
- return new String(output, 0, length);
+ StringReader stringReader = new StringReader(s);
+ Reader foldedStringReader = mappingCharFilterFactory.create(stringReader);
+ try {
+ return IOUtils.toString(foldedStringReader);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
}
return null;
}
[06/50] [abbrv] incubator-unomi git commit: UNOMI-63 Use
ElasticSearch BulkProcessing to perform segment updates - Initial code to
load existing indices - Add code to wait for index startup as there seems to
be some problems with this in a cluster config
Posted by sh...@apache.org.
UNOMI-63 Use ElasticSearch BulkProcessing to perform segment updates
- Initial code to load existing indices
- Add code to wait for index startup as there seems to be some problems with this in a cluster configuration.
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/fcea7057
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/fcea7057
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/fcea7057
Branch: refs/heads/master
Commit: fcea70579e63dea428171754f8113d97dc76ede5
Parents: 9a58214
Author: Serge Huber <sh...@apache.org>
Authored: Thu Nov 17 11:07:54 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Thu Nov 17 11:07:54 2016 +0100
----------------------------------------------------------------------
.../ElasticSearchPersistenceServiceImpl.java | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/fcea7057/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 2208951..2915c32 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -103,6 +103,7 @@ import java.nio.file.Paths;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
@@ -163,6 +164,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private Map<String,String> indexNames;
private List<String> itemsMonthlyIndexed;
private Map<String, String> routingByType;
+ private Set<String> existingIndexNames = new TreeSet<String>();
private String address;
private String port;
@@ -403,6 +405,23 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
bulkProcessor = getBulkProcessor();
}
+ try {
+ IndicesStatsResponse indicesStatsResponse = client.admin().indices().prepareStats().all().execute().get();
+ existingIndexNames = new TreeSet<>(indicesStatsResponse.getIndices().keySet());
+ } catch (InterruptedException e) {
+ logger.error("Error retrieving indices stats", e);
+ } catch (ExecutionException e) {
+ logger.error("Error retrieving indices stats", e);
+ }
+
+ logger.info("Waiting for index creation to complete...");
+
+ client.admin().cluster().prepareHealth()
+ .setWaitForGreenStatus()
+ .get();
+
+ logger.info("Cluster status is GREEN");
+
return null;
}
}.executeInClassLoader();
[39/50] [abbrv] incubator-unomi git commit: - Remove duplicate code -
Add todo to properly use a service tracker/listener to watch for services
appearing/disappearing.
Posted by sh...@apache.org.
- Remove duplicate code
- Add todo to properly use a service tracker/listener to watch for services appearing/disappearing.
Signed-off-by: Serge Huber <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/2ff92868
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/2ff92868
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/2ff92868
Branch: refs/heads/master
Commit: 2ff92868f7b14f22233cbea9fa6af0fe9f7030e3
Parents: 35b9397
Author: Serge Huber <sh...@apache.org>
Authored: Wed Dec 21 22:20:55 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Wed Dec 21 22:20:55 2016 +0100
----------------------------------------------------------------------
.../ElasticSearchPersistenceServiceImpl.java | 12 +-----------
1 file changed, 1 insertion(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/2ff92868/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index dc909b9..f52dc5a 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -463,17 +463,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
if (existingBundle.getBundleContext() != null) {
loadPredefinedMappings(existingBundle.getBundleContext(), true);
}
- if (existingBundle.getRegisteredServices() != null) {
- for (ServiceReference<?> reference : existingBundle.getRegisteredServices()) {
- Object service = bundleContext.getService(reference);
- if (service instanceof ConditionEvaluator) {
- conditionEvaluatorDispatcher.addEvaluator(reference.getProperty("conditionEvaluatorId").toString(), existingBundle.getBundleId(), (ConditionEvaluator) service);
- }
- if (service instanceof ConditionESQueryBuilder) {
- conditionESQueryBuilderDispatcher.addQueryBuilder(reference.getProperty("queryBuilderId").toString(), existingBundle.getBundleId(), (ConditionESQueryBuilder) service);
- }
- }
- }
}
logger.info(this.getClass().getName() + " service started successfully.");
@@ -609,6 +598,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
public void bundleChanged(BundleEvent event) {
switch (event.getType()) {
case BundleEvent.STARTED:
+ // @todo replace this with a proper service tracker/listener
if (event.getBundle() != null && event.getBundle().getRegisteredServices() != null) {
for (ServiceReference<?> reference : event.getBundle().getRegisteredServices()) {
Object service = bundleContext.getService(reference);
[29/50] [abbrv] incubator-unomi git commit: - Introduce a new save
method that supports batching - Use new save method batching to import
Geonames (a lot) faster - Replace slow running ASCIIFoldingFilter call with
faster MappingCharFilter call. See https
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ac2c9ba5/persistence-elasticsearch/core/src/main/resources/mapping-FoldToASCII.txt
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/mapping-FoldToASCII.txt b/persistence-elasticsearch/core/src/main/resources/mapping-FoldToASCII.txt
new file mode 100644
index 0000000..9a84b6e
--- /dev/null
+++ b/persistence-elasticsearch/core/src/main/resources/mapping-FoldToASCII.txt
@@ -0,0 +1,3813 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This map converts alphabetic, numeric, and symbolic Unicode characters
+# which are not in the first 127 ASCII characters (the "Basic Latin" Unicode
+# block) into their ASCII equivalents, if one exists.
+#
+# Characters from the following Unicode blocks are converted; however, only
+# those characters with reasonable ASCII alternatives are converted:
+#
+# - C1 Controls and Latin-1 Supplement: http://www.unicode.org/charts/PDF/U0080.pdf
+# - Latin Extended-A: http://www.unicode.org/charts/PDF/U0100.pdf
+# - Latin Extended-B: http://www.unicode.org/charts/PDF/U0180.pdf
+# - Latin Extended Additional: http://www.unicode.org/charts/PDF/U1E00.pdf
+# - Latin Extended-C: http://www.unicode.org/charts/PDF/U2C60.pdf
+# - Latin Extended-D: http://www.unicode.org/charts/PDF/UA720.pdf
+# - IPA Extensions: http://www.unicode.org/charts/PDF/U0250.pdf
+# - Phonetic Extensions: http://www.unicode.org/charts/PDF/U1D00.pdf
+# - Phonetic Extensions Supplement: http://www.unicode.org/charts/PDF/U1D80.pdf
+# - General Punctuation: http://www.unicode.org/charts/PDF/U2000.pdf
+# - Superscripts and Subscripts: http://www.unicode.org/charts/PDF/U2070.pdf
+# - Enclosed Alphanumerics: http://www.unicode.org/charts/PDF/U2460.pdf
+# - Dingbats: http://www.unicode.org/charts/PDF/U2700.pdf
+# - Supplemental Punctuation: http://www.unicode.org/charts/PDF/U2E00.pdf
+# - Alphabetic Presentation Forms: http://www.unicode.org/charts/PDF/UFB00.pdf
+# - Halfwidth and Fullwidth Forms: http://www.unicode.org/charts/PDF/UFF00.pdf
+#
+# See: http://en.wikipedia.org/wiki/Latin_characters_in_Unicode
+#
+# The set of character conversions supported by this map is a superset of
+# those supported by the map represented by mapping-ISOLatin1Accent.txt.
+#
+# See the bottom of this file for the Perl script used to generate the contents
+# of this file (without this header) from ASCIIFoldingFilter.java.
+
+
+# Syntax:
+# "source" => "target"
+# "source".length() > 0 (source cannot be empty.)
+# "target".length() >= 0 (target can be empty.)
+
+
+# � [LATIN CAPITAL LETTER A WITH GRAVE]
+"\u00C0" => "A"
+
+# � [LATIN CAPITAL LETTER A WITH ACUTE]
+"\u00C1" => "A"
+
+# � [LATIN CAPITAL LETTER A WITH CIRCUMFLEX]
+"\u00C2" => "A"
+
+# � [LATIN CAPITAL LETTER A WITH TILDE]
+"\u00C3" => "A"
+
+# � [LATIN CAPITAL LETTER A WITH DIAERESIS]
+"\u00C4" => "A"
+
+# � [LATIN CAPITAL LETTER A WITH RING ABOVE]
+"\u00C5" => "A"
+
+# \u0100 [LATIN CAPITAL LETTER A WITH MACRON]
+"\u0100" => "A"
+
+# \u0102 [LATIN CAPITAL LETTER A WITH BREVE]
+"\u0102" => "A"
+
+# \u0104 [LATIN CAPITAL LETTER A WITH OGONEK]
+"\u0104" => "A"
+
+# \u018f http://en.wikipedia.org/wiki/Schwa [LATIN CAPITAL LETTER SCHWA]
+"\u018F" => "A"
+
+# \u01cd [LATIN CAPITAL LETTER A WITH CARON]
+"\u01CD" => "A"
+
+# \u01de [LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON]
+"\u01DE" => "A"
+
+# \u01e0 [LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON]
+"\u01E0" => "A"
+
+# \u01fa [LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE]
+"\u01FA" => "A"
+
+# \u0200 [LATIN CAPITAL LETTER A WITH DOUBLE GRAVE]
+"\u0200" => "A"
+
+# \u0202 [LATIN CAPITAL LETTER A WITH INVERTED BREVE]
+"\u0202" => "A"
+
+# \u0226 [LATIN CAPITAL LETTER A WITH DOT ABOVE]
+"\u0226" => "A"
+
+# \u023a [LATIN CAPITAL LETTER A WITH STROKE]
+"\u023A" => "A"
+
+# \u1d00 [LATIN LETTER SMALL CAPITAL A]
+"\u1D00" => "A"
+
+# \u1e00 [LATIN CAPITAL LETTER A WITH RING BELOW]
+"\u1E00" => "A"
+
+# \u1ea0 [LATIN CAPITAL LETTER A WITH DOT BELOW]
+"\u1EA0" => "A"
+
+# \u1ea2 [LATIN CAPITAL LETTER A WITH HOOK ABOVE]
+"\u1EA2" => "A"
+
+# \u1ea4 [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE]
+"\u1EA4" => "A"
+
+# \u1ea6 [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE]
+"\u1EA6" => "A"
+
+# \u1ea8 [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE]
+"\u1EA8" => "A"
+
+# \u1eaa [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE]
+"\u1EAA" => "A"
+
+# \u1eac [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW]
+"\u1EAC" => "A"
+
+# \u1eae [LATIN CAPITAL LETTER A WITH BREVE AND ACUTE]
+"\u1EAE" => "A"
+
+# \u1eb0 [LATIN CAPITAL LETTER A WITH BREVE AND GRAVE]
+"\u1EB0" => "A"
+
+# \u1eb2 [LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE]
+"\u1EB2" => "A"
+
+# \u1eb4 [LATIN CAPITAL LETTER A WITH BREVE AND TILDE]
+"\u1EB4" => "A"
+
+# \u1eb6 [LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW]
+"\u1EB6" => "A"
+
+# \u24b6 [CIRCLED LATIN CAPITAL LETTER A]
+"\u24B6" => "A"
+
+# \uff21 [FULLWIDTH LATIN CAPITAL LETTER A]
+"\uFF21" => "A"
+
+# � [LATIN SMALL LETTER A WITH GRAVE]
+"\u00E0" => "a"
+
+# � [LATIN SMALL LETTER A WITH ACUTE]
+"\u00E1" => "a"
+
+# � [LATIN SMALL LETTER A WITH CIRCUMFLEX]
+"\u00E2" => "a"
+
+# � [LATIN SMALL LETTER A WITH TILDE]
+"\u00E3" => "a"
+
+# � [LATIN SMALL LETTER A WITH DIAERESIS]
+"\u00E4" => "a"
+
+# � [LATIN SMALL LETTER A WITH RING ABOVE]
+"\u00E5" => "a"
+
+# \u0101 [LATIN SMALL LETTER A WITH MACRON]
+"\u0101" => "a"
+
+# \u0103 [LATIN SMALL LETTER A WITH BREVE]
+"\u0103" => "a"
+
+# \u0105 [LATIN SMALL LETTER A WITH OGONEK]
+"\u0105" => "a"
+
+# \u01ce [LATIN SMALL LETTER A WITH CARON]
+"\u01CE" => "a"
+
+# \u01df [LATIN SMALL LETTER A WITH DIAERESIS AND MACRON]
+"\u01DF" => "a"
+
+# \u01e1 [LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON]
+"\u01E1" => "a"
+
+# \u01fb [LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE]
+"\u01FB" => "a"
+
+# \u0201 [LATIN SMALL LETTER A WITH DOUBLE GRAVE]
+"\u0201" => "a"
+
+# \u0203 [LATIN SMALL LETTER A WITH INVERTED BREVE]
+"\u0203" => "a"
+
+# \u0227 [LATIN SMALL LETTER A WITH DOT ABOVE]
+"\u0227" => "a"
+
+# \u0250 [LATIN SMALL LETTER TURNED A]
+"\u0250" => "a"
+
+# \u0259 [LATIN SMALL LETTER SCHWA]
+"\u0259" => "a"
+
+# \u025a [LATIN SMALL LETTER SCHWA WITH HOOK]
+"\u025A" => "a"
+
+# \u1d8f [LATIN SMALL LETTER A WITH RETROFLEX HOOK]
+"\u1D8F" => "a"
+
+# \u1d95 [LATIN SMALL LETTER SCHWA WITH RETROFLEX HOOK]
+"\u1D95" => "a"
+
+# \u1ea1 [LATIN SMALL LETTER A WITH RING BELOW]
+"\u1E01" => "a"
+
+# \u1ea3 [LATIN SMALL LETTER A WITH RIGHT HALF RING]
+"\u1E9A" => "a"
+
+# \u1ea1 [LATIN SMALL LETTER A WITH DOT BELOW]
+"\u1EA1" => "a"
+
+# \u1ea3 [LATIN SMALL LETTER A WITH HOOK ABOVE]
+"\u1EA3" => "a"
+
+# \u1ea5 [LATIN SMALL LETTER A WITH CIRCUMFLEX AND ACUTE]
+"\u1EA5" => "a"
+
+# \u1ea7 [LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE]
+"\u1EA7" => "a"
+
+# \u1ea9 [LATIN SMALL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE]
+"\u1EA9" => "a"
+
+# \u1eab [LATIN SMALL LETTER A WITH CIRCUMFLEX AND TILDE]
+"\u1EAB" => "a"
+
+# \u1ead [LATIN SMALL LETTER A WITH CIRCUMFLEX AND DOT BELOW]
+"\u1EAD" => "a"
+
+# \u1eaf [LATIN SMALL LETTER A WITH BREVE AND ACUTE]
+"\u1EAF" => "a"
+
+# \u1eb1 [LATIN SMALL LETTER A WITH BREVE AND GRAVE]
+"\u1EB1" => "a"
+
+# \u1eb3 [LATIN SMALL LETTER A WITH BREVE AND HOOK ABOVE]
+"\u1EB3" => "a"
+
+# \u1eb5 [LATIN SMALL LETTER A WITH BREVE AND TILDE]
+"\u1EB5" => "a"
+
+# \u1eb7 [LATIN SMALL LETTER A WITH BREVE AND DOT BELOW]
+"\u1EB7" => "a"
+
+# \u2090 [LATIN SUBSCRIPT SMALL LETTER A]
+"\u2090" => "a"
+
+# \u2094 [LATIN SUBSCRIPT SMALL LETTER SCHWA]
+"\u2094" => "a"
+
+# \u24d0 [CIRCLED LATIN SMALL LETTER A]
+"\u24D0" => "a"
+
+# \u2c65 [LATIN SMALL LETTER A WITH STROKE]
+"\u2C65" => "a"
+
+# \u2c6f [LATIN CAPITAL LETTER TURNED A]
+"\u2C6F" => "a"
+
+# \uff41 [FULLWIDTH LATIN SMALL LETTER A]
+"\uFF41" => "a"
+
+# \ua732 [LATIN CAPITAL LETTER AA]
+"\uA732" => "AA"
+
+# � [LATIN CAPITAL LETTER AE]
+"\u00C6" => "AE"
+
+# \u01e2 [LATIN CAPITAL LETTER AE WITH MACRON]
+"\u01E2" => "AE"
+
+# \u01fc [LATIN CAPITAL LETTER AE WITH ACUTE]
+"\u01FC" => "AE"
+
+# \u1d01 [LATIN LETTER SMALL CAPITAL AE]
+"\u1D01" => "AE"
+
+# \ua734 [LATIN CAPITAL LETTER AO]
+"\uA734" => "AO"
+
+# \ua736 [LATIN CAPITAL LETTER AU]
+"\uA736" => "AU"
+
+# \ua738 [LATIN CAPITAL LETTER AV]
+"\uA738" => "AV"
+
+# \ua73a [LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR]
+"\uA73A" => "AV"
+
+# \ua73c [LATIN CAPITAL LETTER AY]
+"\uA73C" => "AY"
+
+# \u249c [PARENTHESIZED LATIN SMALL LETTER A]
+"\u249C" => "(a)"
+
+# \ua733 [LATIN SMALL LETTER AA]
+"\uA733" => "aa"
+
+# � [LATIN SMALL LETTER AE]
+"\u00E6" => "ae"
+
+# \u01e3 [LATIN SMALL LETTER AE WITH MACRON]
+"\u01E3" => "ae"
+
+# \u01fd [LATIN SMALL LETTER AE WITH ACUTE]
+"\u01FD" => "ae"
+
+# \u1d02 [LATIN SMALL LETTER TURNED AE]
+"\u1D02" => "ae"
+
+# \ua735 [LATIN SMALL LETTER AO]
+"\uA735" => "ao"
+
+# \ua737 [LATIN SMALL LETTER AU]
+"\uA737" => "au"
+
+# \ua739 [LATIN SMALL LETTER AV]
+"\uA739" => "av"
+
+# \ua73b [LATIN SMALL LETTER AV WITH HORIZONTAL BAR]
+"\uA73B" => "av"
+
+# \ua73d [LATIN SMALL LETTER AY]
+"\uA73D" => "ay"
+
+# \u0181 [LATIN CAPITAL LETTER B WITH HOOK]
+"\u0181" => "B"
+
+# \u0182 [LATIN CAPITAL LETTER B WITH TOPBAR]
+"\u0182" => "B"
+
+# \u0243 [LATIN CAPITAL LETTER B WITH STROKE]
+"\u0243" => "B"
+
+# \u0299 [LATIN LETTER SMALL CAPITAL B]
+"\u0299" => "B"
+
+# \u1d03 [LATIN LETTER SMALL CAPITAL BARRED B]
+"\u1D03" => "B"
+
+# \u1e02 [LATIN CAPITAL LETTER B WITH DOT ABOVE]
+"\u1E02" => "B"
+
+# \u1e04 [LATIN CAPITAL LETTER B WITH DOT BELOW]
+"\u1E04" => "B"
+
+# \u1e06 [LATIN CAPITAL LETTER B WITH LINE BELOW]
+"\u1E06" => "B"
+
+# \u24b7 [CIRCLED LATIN CAPITAL LETTER B]
+"\u24B7" => "B"
+
+# \uff22 [FULLWIDTH LATIN CAPITAL LETTER B]
+"\uFF22" => "B"
+
+# \u0180 [LATIN SMALL LETTER B WITH STROKE]
+"\u0180" => "b"
+
+# \u0183 [LATIN SMALL LETTER B WITH TOPBAR]
+"\u0183" => "b"
+
+# \u0253 [LATIN SMALL LETTER B WITH HOOK]
+"\u0253" => "b"
+
+# \u1d6c [LATIN SMALL LETTER B WITH MIDDLE TILDE]
+"\u1D6C" => "b"
+
+# \u1d80 [LATIN SMALL LETTER B WITH PALATAL HOOK]
+"\u1D80" => "b"
+
+# \u1e03 [LATIN SMALL LETTER B WITH DOT ABOVE]
+"\u1E03" => "b"
+
+# \u1e05 [LATIN SMALL LETTER B WITH DOT BELOW]
+"\u1E05" => "b"
+
+# \u1e07 [LATIN SMALL LETTER B WITH LINE BELOW]
+"\u1E07" => "b"
+
+# \u24d1 [CIRCLED LATIN SMALL LETTER B]
+"\u24D1" => "b"
+
+# \uff42 [FULLWIDTH LATIN SMALL LETTER B]
+"\uFF42" => "b"
+
+# \u249d [PARENTHESIZED LATIN SMALL LETTER B]
+"\u249D" => "(b)"
+
+# � [LATIN CAPITAL LETTER C WITH CEDILLA]
+"\u00C7" => "C"
+
+# \u0106 [LATIN CAPITAL LETTER C WITH ACUTE]
+"\u0106" => "C"
+
+# \u0108 [LATIN CAPITAL LETTER C WITH CIRCUMFLEX]
+"\u0108" => "C"
+
+# \u010a [LATIN CAPITAL LETTER C WITH DOT ABOVE]
+"\u010A" => "C"
+
+# \u010c [LATIN CAPITAL LETTER C WITH CARON]
+"\u010C" => "C"
+
+# \u0187 [LATIN CAPITAL LETTER C WITH HOOK]
+"\u0187" => "C"
+
+# \u023b [LATIN CAPITAL LETTER C WITH STROKE]
+"\u023B" => "C"
+
+# \u0297 [LATIN LETTER STRETCHED C]
+"\u0297" => "C"
+
+# \u1d04 [LATIN LETTER SMALL CAPITAL C]
+"\u1D04" => "C"
+
+# \u1e08 [LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE]
+"\u1E08" => "C"
+
+# \u24b8 [CIRCLED LATIN CAPITAL LETTER C]
+"\u24B8" => "C"
+
+# \uff23 [FULLWIDTH LATIN CAPITAL LETTER C]
+"\uFF23" => "C"
+
+# � [LATIN SMALL LETTER C WITH CEDILLA]
+"\u00E7" => "c"
+
+# \u0107 [LATIN SMALL LETTER C WITH ACUTE]
+"\u0107" => "c"
+
+# \u0109 [LATIN SMALL LETTER C WITH CIRCUMFLEX]
+"\u0109" => "c"
+
+# \u010b [LATIN SMALL LETTER C WITH DOT ABOVE]
+"\u010B" => "c"
+
+# \u010d [LATIN SMALL LETTER C WITH CARON]
+"\u010D" => "c"
+
+# \u0188 [LATIN SMALL LETTER C WITH HOOK]
+"\u0188" => "c"
+
+# \u023c [LATIN SMALL LETTER C WITH STROKE]
+"\u023C" => "c"
+
+# \u0255 [LATIN SMALL LETTER C WITH CURL]
+"\u0255" => "c"
+
+# \u1e09 [LATIN SMALL LETTER C WITH CEDILLA AND ACUTE]
+"\u1E09" => "c"
+
+# \u2184 [LATIN SMALL LETTER REVERSED C]
+"\u2184" => "c"
+
+# \u24d2 [CIRCLED LATIN SMALL LETTER C]
+"\u24D2" => "c"
+
+# \ua73e [LATIN CAPITAL LETTER REVERSED C WITH DOT]
+"\uA73E" => "c"
+
+# \ua73f [LATIN SMALL LETTER REVERSED C WITH DOT]
+"\uA73F" => "c"
+
+# \uff43 [FULLWIDTH LATIN SMALL LETTER C]
+"\uFF43" => "c"
+
+# \u249e [PARENTHESIZED LATIN SMALL LETTER C]
+"\u249E" => "(c)"
+
+# � [LATIN CAPITAL LETTER ETH]
+"\u00D0" => "D"
+
+# \u010e [LATIN CAPITAL LETTER D WITH CARON]
+"\u010E" => "D"
+
+# \u0110 [LATIN CAPITAL LETTER D WITH STROKE]
+"\u0110" => "D"
+
+# \u0189 [LATIN CAPITAL LETTER AFRICAN D]
+"\u0189" => "D"
+
+# \u018a [LATIN CAPITAL LETTER D WITH HOOK]
+"\u018A" => "D"
+
+# \u018b [LATIN CAPITAL LETTER D WITH TOPBAR]
+"\u018B" => "D"
+
+# \u1d05 [LATIN LETTER SMALL CAPITAL D]
+"\u1D05" => "D"
+
+# \u1d06 [LATIN LETTER SMALL CAPITAL ETH]
+"\u1D06" => "D"
+
+# \u1e0a [LATIN CAPITAL LETTER D WITH DOT ABOVE]
+"\u1E0A" => "D"
+
+# \u1e0c [LATIN CAPITAL LETTER D WITH DOT BELOW]
+"\u1E0C" => "D"
+
+# \u1e0e [LATIN CAPITAL LETTER D WITH LINE BELOW]
+"\u1E0E" => "D"
+
+# \u1e10 [LATIN CAPITAL LETTER D WITH CEDILLA]
+"\u1E10" => "D"
+
+# \u1e12 [LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW]
+"\u1E12" => "D"
+
+# \u24b9 [CIRCLED LATIN CAPITAL LETTER D]
+"\u24B9" => "D"
+
+# \ua779 [LATIN CAPITAL LETTER INSULAR D]
+"\uA779" => "D"
+
+# \uff24 [FULLWIDTH LATIN CAPITAL LETTER D]
+"\uFF24" => "D"
+
+# � [LATIN SMALL LETTER ETH]
+"\u00F0" => "d"
+
+# \u010f [LATIN SMALL LETTER D WITH CARON]
+"\u010F" => "d"
+
+# \u0111 [LATIN SMALL LETTER D WITH STROKE]
+"\u0111" => "d"
+
+# \u018c [LATIN SMALL LETTER D WITH TOPBAR]
+"\u018C" => "d"
+
+# \u0221 [LATIN SMALL LETTER D WITH CURL]
+"\u0221" => "d"
+
+# \u0256 [LATIN SMALL LETTER D WITH TAIL]
+"\u0256" => "d"
+
+# \u0257 [LATIN SMALL LETTER D WITH HOOK]
+"\u0257" => "d"
+
+# \u1d6d [LATIN SMALL LETTER D WITH MIDDLE TILDE]
+"\u1D6D" => "d"
+
+# \u1d81 [LATIN SMALL LETTER D WITH PALATAL HOOK]
+"\u1D81" => "d"
+
+# \u1d91 [LATIN SMALL LETTER D WITH HOOK AND TAIL]
+"\u1D91" => "d"
+
+# \u1e0b [LATIN SMALL LETTER D WITH DOT ABOVE]
+"\u1E0B" => "d"
+
+# \u1e0d [LATIN SMALL LETTER D WITH DOT BELOW]
+"\u1E0D" => "d"
+
+# \u1e0f [LATIN SMALL LETTER D WITH LINE BELOW]
+"\u1E0F" => "d"
+
+# \u1e11 [LATIN SMALL LETTER D WITH CEDILLA]
+"\u1E11" => "d"
+
+# \u1e13 [LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW]
+"\u1E13" => "d"
+
+# \u24d3 [CIRCLED LATIN SMALL LETTER D]
+"\u24D3" => "d"
+
+# \ua77a [LATIN SMALL LETTER INSULAR D]
+"\uA77A" => "d"
+
+# \uff44 [FULLWIDTH LATIN SMALL LETTER D]
+"\uFF44" => "d"
+
+# \u01c4 [LATIN CAPITAL LETTER DZ WITH CARON]
+"\u01C4" => "DZ"
+
+# \u01f1 [LATIN CAPITAL LETTER DZ]
+"\u01F1" => "DZ"
+
+# \u01c5 [LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON]
+"\u01C5" => "Dz"
+
+# \u01f2 [LATIN CAPITAL LETTER D WITH SMALL LETTER Z]
+"\u01F2" => "Dz"
+
+# \u249f [PARENTHESIZED LATIN SMALL LETTER D]
+"\u249F" => "(d)"
+
+# \u0238 [LATIN SMALL LETTER DB DIGRAPH]
+"\u0238" => "db"
+
+# \u01c6 [LATIN SMALL LETTER DZ WITH CARON]
+"\u01C6" => "dz"
+
+# \u01f3 [LATIN SMALL LETTER DZ]
+"\u01F3" => "dz"
+
+# \u02a3 [LATIN SMALL LETTER DZ DIGRAPH]
+"\u02A3" => "dz"
+
+# \u02a5 [LATIN SMALL LETTER DZ DIGRAPH WITH CURL]
+"\u02A5" => "dz"
+
+# � [LATIN CAPITAL LETTER E WITH GRAVE]
+"\u00C8" => "E"
+
+# � [LATIN CAPITAL LETTER E WITH ACUTE]
+"\u00C9" => "E"
+
+# � [LATIN CAPITAL LETTER E WITH CIRCUMFLEX]
+"\u00CA" => "E"
+
+# � [LATIN CAPITAL LETTER E WITH DIAERESIS]
+"\u00CB" => "E"
+
+# \u0112 [LATIN CAPITAL LETTER E WITH MACRON]
+"\u0112" => "E"
+
+# \u0114 [LATIN CAPITAL LETTER E WITH BREVE]
+"\u0114" => "E"
+
+# \u0116 [LATIN CAPITAL LETTER E WITH DOT ABOVE]
+"\u0116" => "E"
+
+# \u0118 [LATIN CAPITAL LETTER E WITH OGONEK]
+"\u0118" => "E"
+
+# \u011a [LATIN CAPITAL LETTER E WITH CARON]
+"\u011A" => "E"
+
+# \u018e [LATIN CAPITAL LETTER REVERSED E]
+"\u018E" => "E"
+
+# \u0190 [LATIN CAPITAL LETTER OPEN E]
+"\u0190" => "E"
+
+# \u0204 [LATIN CAPITAL LETTER E WITH DOUBLE GRAVE]
+"\u0204" => "E"
+
+# \u0206 [LATIN CAPITAL LETTER E WITH INVERTED BREVE]
+"\u0206" => "E"
+
+# \u0228 [LATIN CAPITAL LETTER E WITH CEDILLA]
+"\u0228" => "E"
+
+# \u0246 [LATIN CAPITAL LETTER E WITH STROKE]
+"\u0246" => "E"
+
+# \u1d07 [LATIN LETTER SMALL CAPITAL E]
+"\u1D07" => "E"
+
+# \u1e14 [LATIN CAPITAL LETTER E WITH MACRON AND GRAVE]
+"\u1E14" => "E"
+
+# \u1e16 [LATIN CAPITAL LETTER E WITH MACRON AND ACUTE]
+"\u1E16" => "E"
+
+# \u1e18 [LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW]
+"\u1E18" => "E"
+
+# \u1e1a [LATIN CAPITAL LETTER E WITH TILDE BELOW]
+"\u1E1A" => "E"
+
+# \u1e1c [LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE]
+"\u1E1C" => "E"
+
+# \u1eb8 [LATIN CAPITAL LETTER E WITH DOT BELOW]
+"\u1EB8" => "E"
+
+# \u1eba [LATIN CAPITAL LETTER E WITH HOOK ABOVE]
+"\u1EBA" => "E"
+
+# \u1ebc [LATIN CAPITAL LETTER E WITH TILDE]
+"\u1EBC" => "E"
+
+# \u1ebe [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE]
+"\u1EBE" => "E"
+
+# \u1ec0 [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE]
+"\u1EC0" => "E"
+
+# \u1ec2 [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE]
+"\u1EC2" => "E"
+
+# \u1ec4 [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE]
+"\u1EC4" => "E"
+
+# \u1ec6 [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW]
+"\u1EC6" => "E"
+
+# \u24ba [CIRCLED LATIN CAPITAL LETTER E]
+"\u24BA" => "E"
+
+# \u2c7b [LATIN LETTER SMALL CAPITAL TURNED E]
+"\u2C7B" => "E"
+
+# \uff25 [FULLWIDTH LATIN CAPITAL LETTER E]
+"\uFF25" => "E"
+
+# � [LATIN SMALL LETTER E WITH GRAVE]
+"\u00E8" => "e"
+
+# � [LATIN SMALL LETTER E WITH ACUTE]
+"\u00E9" => "e"
+
+# � [LATIN SMALL LETTER E WITH CIRCUMFLEX]
+"\u00EA" => "e"
+
+# � [LATIN SMALL LETTER E WITH DIAERESIS]
+"\u00EB" => "e"
+
+# \u0113 [LATIN SMALL LETTER E WITH MACRON]
+"\u0113" => "e"
+
+# \u0115 [LATIN SMALL LETTER E WITH BREVE]
+"\u0115" => "e"
+
+# \u0117 [LATIN SMALL LETTER E WITH DOT ABOVE]
+"\u0117" => "e"
+
+# \u0119 [LATIN SMALL LETTER E WITH OGONEK]
+"\u0119" => "e"
+
+# \u011b [LATIN SMALL LETTER E WITH CARON]
+"\u011B" => "e"
+
+# \u01dd [LATIN SMALL LETTER TURNED E]
+"\u01DD" => "e"
+
+# \u0205 [LATIN SMALL LETTER E WITH DOUBLE GRAVE]
+"\u0205" => "e"
+
+# \u0207 [LATIN SMALL LETTER E WITH INVERTED BREVE]
+"\u0207" => "e"
+
+# \u0229 [LATIN SMALL LETTER E WITH CEDILLA]
+"\u0229" => "e"
+
+# \u0247 [LATIN SMALL LETTER E WITH STROKE]
+"\u0247" => "e"
+
+# \u0258 [LATIN SMALL LETTER REVERSED E]
+"\u0258" => "e"
+
+# \u025b [LATIN SMALL LETTER OPEN E]
+"\u025B" => "e"
+
+# \u025c [LATIN SMALL LETTER REVERSED OPEN E]
+"\u025C" => "e"
+
+# \u025d [LATIN SMALL LETTER REVERSED OPEN E WITH HOOK]
+"\u025D" => "e"
+
+# \u025e [LATIN SMALL LETTER CLOSED REVERSED OPEN E]
+"\u025E" => "e"
+
+# \u029a [LATIN SMALL LETTER CLOSED OPEN E]
+"\u029A" => "e"
+
+# \u1d08 [LATIN SMALL LETTER TURNED OPEN E]
+"\u1D08" => "e"
+
+# \u1d92 [LATIN SMALL LETTER E WITH RETROFLEX HOOK]
+"\u1D92" => "e"
+
+# \u1d93 [LATIN SMALL LETTER OPEN E WITH RETROFLEX HOOK]
+"\u1D93" => "e"
+
+# \u1d94 [LATIN SMALL LETTER REVERSED OPEN E WITH RETROFLEX HOOK]
+"\u1D94" => "e"
+
+# \u1e15 [LATIN SMALL LETTER E WITH MACRON AND GRAVE]
+"\u1E15" => "e"
+
+# \u1e17 [LATIN SMALL LETTER E WITH MACRON AND ACUTE]
+"\u1E17" => "e"
+
+# \u1e19 [LATIN SMALL LETTER E WITH CIRCUMFLEX BELOW]
+"\u1E19" => "e"
+
+# \u1e1b [LATIN SMALL LETTER E WITH TILDE BELOW]
+"\u1E1B" => "e"
+
+# \u1e1d [LATIN SMALL LETTER E WITH CEDILLA AND BREVE]
+"\u1E1D" => "e"
+
+# \u1eb9 [LATIN SMALL LETTER E WITH DOT BELOW]
+"\u1EB9" => "e"
+
+# \u1ebb [LATIN SMALL LETTER E WITH HOOK ABOVE]
+"\u1EBB" => "e"
+
+# \u1ebd [LATIN SMALL LETTER E WITH TILDE]
+"\u1EBD" => "e"
+
+# \u1ebf [LATIN SMALL LETTER E WITH CIRCUMFLEX AND ACUTE]
+"\u1EBF" => "e"
+
+# \u1ec1 [LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE]
+"\u1EC1" => "e"
+
+# \u1ec3 [LATIN SMALL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE]
+"\u1EC3" => "e"
+
+# \u1ec5 [LATIN SMALL LETTER E WITH CIRCUMFLEX AND TILDE]
+"\u1EC5" => "e"
+
+# \u1ec7 [LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW]
+"\u1EC7" => "e"
+
+# \u2091 [LATIN SUBSCRIPT SMALL LETTER E]
+"\u2091" => "e"
+
+# \u24d4 [CIRCLED LATIN SMALL LETTER E]
+"\u24D4" => "e"
+
+# \u2c78 [LATIN SMALL LETTER E WITH NOTCH]
+"\u2C78" => "e"
+
+# \uff45 [FULLWIDTH LATIN SMALL LETTER E]
+"\uFF45" => "e"
+
+# \u24a0 [PARENTHESIZED LATIN SMALL LETTER E]
+"\u24A0" => "(e)"
+
+# \u0191 [LATIN CAPITAL LETTER F WITH HOOK]
+"\u0191" => "F"
+
+# \u1e1e [LATIN CAPITAL LETTER F WITH DOT ABOVE]
+"\u1E1E" => "F"
+
+# \u24bb [CIRCLED LATIN CAPITAL LETTER F]
+"\u24BB" => "F"
+
+# \ua730 [LATIN LETTER SMALL CAPITAL F]
+"\uA730" => "F"
+
+# \ua77b [LATIN CAPITAL LETTER INSULAR F]
+"\uA77B" => "F"
+
+# \ua7fb [LATIN EPIGRAPHIC LETTER REVERSED F]
+"\uA7FB" => "F"
+
+# \uff26 [FULLWIDTH LATIN CAPITAL LETTER F]
+"\uFF26" => "F"
+
+# \u0192 [LATIN SMALL LETTER F WITH HOOK]
+"\u0192" => "f"
+
+# \u1d6e [LATIN SMALL LETTER F WITH MIDDLE TILDE]
+"\u1D6E" => "f"
+
+# \u1d82 [LATIN SMALL LETTER F WITH PALATAL HOOK]
+"\u1D82" => "f"
+
+# \u1e1f [LATIN SMALL LETTER F WITH DOT ABOVE]
+"\u1E1F" => "f"
+
+# \u1e9b [LATIN SMALL LETTER LONG S WITH DOT ABOVE]
+"\u1E9B" => "f"
+
+# \u24d5 [CIRCLED LATIN SMALL LETTER F]
+"\u24D5" => "f"
+
+# \ua77c [LATIN SMALL LETTER INSULAR F]
+"\uA77C" => "f"
+
+# \uff46 [FULLWIDTH LATIN SMALL LETTER F]
+"\uFF46" => "f"
+
+# \u24a1 [PARENTHESIZED LATIN SMALL LETTER F]
+"\u24A1" => "(f)"
+
+# \ufb00 [LATIN SMALL LIGATURE FF]
+"\uFB00" => "ff"
+
+# \ufb03 [LATIN SMALL LIGATURE FFI]
+"\uFB03" => "ffi"
+
+# \ufb04 [LATIN SMALL LIGATURE FFL]
+"\uFB04" => "ffl"
+
+# \ufb01 [LATIN SMALL LIGATURE FI]
+"\uFB01" => "fi"
+
+# \ufb02 [LATIN SMALL LIGATURE FL]
+"\uFB02" => "fl"
+
+# \u011c [LATIN CAPITAL LETTER G WITH CIRCUMFLEX]
+"\u011C" => "G"
+
+# \u011e [LATIN CAPITAL LETTER G WITH BREVE]
+"\u011E" => "G"
+
+# \u0120 [LATIN CAPITAL LETTER G WITH DOT ABOVE]
+"\u0120" => "G"
+
+# \u0122 [LATIN CAPITAL LETTER G WITH CEDILLA]
+"\u0122" => "G"
+
+# \u0193 [LATIN CAPITAL LETTER G WITH HOOK]
+"\u0193" => "G"
+
+# \u01e4 [LATIN CAPITAL LETTER G WITH STROKE]
+"\u01E4" => "G"
+
+# \u01e5 [LATIN SMALL LETTER G WITH STROKE]
+"\u01E5" => "G"
+
+# \u01e6 [LATIN CAPITAL LETTER G WITH CARON]
+"\u01E6" => "G"
+
+# \u01e7 [LATIN SMALL LETTER G WITH CARON]
+"\u01E7" => "G"
+
+# \u01f4 [LATIN CAPITAL LETTER G WITH ACUTE]
+"\u01F4" => "G"
+
+# \u0262 [LATIN LETTER SMALL CAPITAL G]
+"\u0262" => "G"
+
+# \u029b [LATIN LETTER SMALL CAPITAL G WITH HOOK]
+"\u029B" => "G"
+
+# \u1e20 [LATIN CAPITAL LETTER G WITH MACRON]
+"\u1E20" => "G"
+
+# \u24bc [CIRCLED LATIN CAPITAL LETTER G]
+"\u24BC" => "G"
+
+# \ua77d [LATIN CAPITAL LETTER INSULAR G]
+"\uA77D" => "G"
+
+# \ua77e [LATIN CAPITAL LETTER TURNED INSULAR G]
+"\uA77E" => "G"
+
+# \uff27 [FULLWIDTH LATIN CAPITAL LETTER G]
+"\uFF27" => "G"
+
+# \u011d [LATIN SMALL LETTER G WITH CIRCUMFLEX]
+"\u011D" => "g"
+
+# \u011f [LATIN SMALL LETTER G WITH BREVE]
+"\u011F" => "g"
+
+# \u0121 [LATIN SMALL LETTER G WITH DOT ABOVE]
+"\u0121" => "g"
+
+# \u0123 [LATIN SMALL LETTER G WITH CEDILLA]
+"\u0123" => "g"
+
+# \u01f5 [LATIN SMALL LETTER G WITH ACUTE]
+"\u01F5" => "g"
+
+# \u0260 [LATIN SMALL LETTER G WITH HOOK]
+"\u0260" => "g"
+
+# \u0261 [LATIN SMALL LETTER SCRIPT G]
+"\u0261" => "g"
+
+# \u1d77 [LATIN SMALL LETTER TURNED G]
+"\u1D77" => "g"
+
+# \u1d79 [LATIN SMALL LETTER INSULAR G]
+"\u1D79" => "g"
+
+# \u1d83 [LATIN SMALL LETTER G WITH PALATAL HOOK]
+"\u1D83" => "g"
+
+# \u1e21 [LATIN SMALL LETTER G WITH MACRON]
+"\u1E21" => "g"
+
+# \u24d6 [CIRCLED LATIN SMALL LETTER G]
+"\u24D6" => "g"
+
+# \ua77f [LATIN SMALL LETTER TURNED INSULAR G]
+"\uA77F" => "g"
+
+# \uff47 [FULLWIDTH LATIN SMALL LETTER G]
+"\uFF47" => "g"
+
+# \u24a2 [PARENTHESIZED LATIN SMALL LETTER G]
+"\u24A2" => "(g)"
+
+# \u0124 [LATIN CAPITAL LETTER H WITH CIRCUMFLEX]
+"\u0124" => "H"
+
+# \u0126 [LATIN CAPITAL LETTER H WITH STROKE]
+"\u0126" => "H"
+
+# \u021e [LATIN CAPITAL LETTER H WITH CARON]
+"\u021E" => "H"
+
+# \u029c [LATIN LETTER SMALL CAPITAL H]
+"\u029C" => "H"
+
+# \u1e22 [LATIN CAPITAL LETTER H WITH DOT ABOVE]
+"\u1E22" => "H"
+
+# \u1e24 [LATIN CAPITAL LETTER H WITH DOT BELOW]
+"\u1E24" => "H"
+
+# \u1e26 [LATIN CAPITAL LETTER H WITH DIAERESIS]
+"\u1E26" => "H"
+
+# \u1e28 [LATIN CAPITAL LETTER H WITH CEDILLA]
+"\u1E28" => "H"
+
+# \u1e2a [LATIN CAPITAL LETTER H WITH BREVE BELOW]
+"\u1E2A" => "H"
+
+# \u24bd [CIRCLED LATIN CAPITAL LETTER H]
+"\u24BD" => "H"
+
+# \u2c67 [LATIN CAPITAL LETTER H WITH DESCENDER]
+"\u2C67" => "H"
+
+# \u2c75 [LATIN CAPITAL LETTER HALF H]
+"\u2C75" => "H"
+
+# \uff28 [FULLWIDTH LATIN CAPITAL LETTER H]
+"\uFF28" => "H"
+
+# \u0125 [LATIN SMALL LETTER H WITH CIRCUMFLEX]
+"\u0125" => "h"
+
+# \u0127 [LATIN SMALL LETTER H WITH STROKE]
+"\u0127" => "h"
+
+# \u021f [LATIN SMALL LETTER H WITH CARON]
+"\u021F" => "h"
+
+# \u0265 [LATIN SMALL LETTER TURNED H]
+"\u0265" => "h"
+
+# \u0266 [LATIN SMALL LETTER H WITH HOOK]
+"\u0266" => "h"
+
+# \u02ae [LATIN SMALL LETTER TURNED H WITH FISHHOOK]
+"\u02AE" => "h"
+
+# \u02af [LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL]
+"\u02AF" => "h"
+
+# \u1e23 [LATIN SMALL LETTER H WITH DOT ABOVE]
+"\u1E23" => "h"
+
+# \u1e25 [LATIN SMALL LETTER H WITH DOT BELOW]
+"\u1E25" => "h"
+
+# \u1e27 [LATIN SMALL LETTER H WITH DIAERESIS]
+"\u1E27" => "h"
+
+# \u1e29 [LATIN SMALL LETTER H WITH CEDILLA]
+"\u1E29" => "h"
+
+# \u1e2b [LATIN SMALL LETTER H WITH BREVE BELOW]
+"\u1E2B" => "h"
+
+# \u1e96 [LATIN SMALL LETTER H WITH LINE BELOW]
+"\u1E96" => "h"
+
+# \u24d7 [CIRCLED LATIN SMALL LETTER H]
+"\u24D7" => "h"
+
+# \u2c68 [LATIN SMALL LETTER H WITH DESCENDER]
+"\u2C68" => "h"
+
+# \u2c76 [LATIN SMALL LETTER HALF H]
+"\u2C76" => "h"
+
+# \uff48 [FULLWIDTH LATIN SMALL LETTER H]
+"\uFF48" => "h"
+
+# \u01f6 http://en.wikipedia.org/wiki/Hwair [LATIN CAPITAL LETTER HWAIR]
+"\u01F6" => "HV"
+
+# \u24a3 [PARENTHESIZED LATIN SMALL LETTER H]
+"\u24A3" => "(h)"
+
+# \u0195 [LATIN SMALL LETTER HV]
+"\u0195" => "hv"
+
+# � [LATIN CAPITAL LETTER I WITH GRAVE]
+"\u00CC" => "I"
+
+# � [LATIN CAPITAL LETTER I WITH ACUTE]
+"\u00CD" => "I"
+
+# � [LATIN CAPITAL LETTER I WITH CIRCUMFLEX]
+"\u00CE" => "I"
+
+# � [LATIN CAPITAL LETTER I WITH DIAERESIS]
+"\u00CF" => "I"
+
+# \u0128 [LATIN CAPITAL LETTER I WITH TILDE]
+"\u0128" => "I"
+
+# \u012a [LATIN CAPITAL LETTER I WITH MACRON]
+"\u012A" => "I"
+
+# \u012c [LATIN CAPITAL LETTER I WITH BREVE]
+"\u012C" => "I"
+
+# \u012e [LATIN CAPITAL LETTER I WITH OGONEK]
+"\u012E" => "I"
+
+# \u0130 [LATIN CAPITAL LETTER I WITH DOT ABOVE]
+"\u0130" => "I"
+
+# \u0196 [LATIN CAPITAL LETTER IOTA]
+"\u0196" => "I"
+
+# \u0197 [LATIN CAPITAL LETTER I WITH STROKE]
+"\u0197" => "I"
+
+# \u01cf [LATIN CAPITAL LETTER I WITH CARON]
+"\u01CF" => "I"
+
+# \u0208 [LATIN CAPITAL LETTER I WITH DOUBLE GRAVE]
+"\u0208" => "I"
+
+# \u020a [LATIN CAPITAL LETTER I WITH INVERTED BREVE]
+"\u020A" => "I"
+
+# \u026a [LATIN LETTER SMALL CAPITAL I]
+"\u026A" => "I"
+
+# \u1d7b [LATIN SMALL CAPITAL LETTER I WITH STROKE]
+"\u1D7B" => "I"
+
+# \u1e2c [LATIN CAPITAL LETTER I WITH TILDE BELOW]
+"\u1E2C" => "I"
+
+# \u1e2e [LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE]
+"\u1E2E" => "I"
+
+# \u1ec8 [LATIN CAPITAL LETTER I WITH HOOK ABOVE]
+"\u1EC8" => "I"
+
+# \u1eca [LATIN CAPITAL LETTER I WITH DOT BELOW]
+"\u1ECA" => "I"
+
+# \u24be [CIRCLED LATIN CAPITAL LETTER I]
+"\u24BE" => "I"
+
+# \ua7fe [LATIN EPIGRAPHIC LETTER I LONGA]
+"\uA7FE" => "I"
+
+# \uff29 [FULLWIDTH LATIN CAPITAL LETTER I]
+"\uFF29" => "I"
+
+# � [LATIN SMALL LETTER I WITH GRAVE]
+"\u00EC" => "i"
+
+# � [LATIN SMALL LETTER I WITH ACUTE]
+"\u00ED" => "i"
+
+# � [LATIN SMALL LETTER I WITH CIRCUMFLEX]
+"\u00EE" => "i"
+
+# � [LATIN SMALL LETTER I WITH DIAERESIS]
+"\u00EF" => "i"
+
+# \u0129 [LATIN SMALL LETTER I WITH TILDE]
+"\u0129" => "i"
+
+# \u012b [LATIN SMALL LETTER I WITH MACRON]
+"\u012B" => "i"
+
+# \u012d [LATIN SMALL LETTER I WITH BREVE]
+"\u012D" => "i"
+
+# \u012f [LATIN SMALL LETTER I WITH OGONEK]
+"\u012F" => "i"
+
+# \u0131 [LATIN SMALL LETTER DOTLESS I]
+"\u0131" => "i"
+
+# \u01d0 [LATIN SMALL LETTER I WITH CARON]
+"\u01D0" => "i"
+
+# \u0209 [LATIN SMALL LETTER I WITH DOUBLE GRAVE]
+"\u0209" => "i"
+
+# \u020b [LATIN SMALL LETTER I WITH INVERTED BREVE]
+"\u020B" => "i"
+
+# \u0268 [LATIN SMALL LETTER I WITH STROKE]
+"\u0268" => "i"
+
+# \u1d09 [LATIN SMALL LETTER TURNED I]
+"\u1D09" => "i"
+
+# \u1d62 [LATIN SUBSCRIPT SMALL LETTER I]
+"\u1D62" => "i"
+
+# \u1d7c [LATIN SMALL LETTER IOTA WITH STROKE]
+"\u1D7C" => "i"
+
+# \u1d96 [LATIN SMALL LETTER I WITH RETROFLEX HOOK]
+"\u1D96" => "i"
+
+# \u1e2d [LATIN SMALL LETTER I WITH TILDE BELOW]
+"\u1E2D" => "i"
+
+# \u1e2f [LATIN SMALL LETTER I WITH DIAERESIS AND ACUTE]
+"\u1E2F" => "i"
+
+# \u1ec9 [LATIN SMALL LETTER I WITH HOOK ABOVE]
+"\u1EC9" => "i"
+
+# \u1ecb [LATIN SMALL LETTER I WITH DOT BELOW]
+"\u1ECB" => "i"
+
+# \u2071 [SUPERSCRIPT LATIN SMALL LETTER I]
+"\u2071" => "i"
+
+# \u24d8 [CIRCLED LATIN SMALL LETTER I]
+"\u24D8" => "i"
+
+# \uff49 [FULLWIDTH LATIN SMALL LETTER I]
+"\uFF49" => "i"
+
+# \u0132 [LATIN CAPITAL LIGATURE IJ]
+"\u0132" => "IJ"
+
+# \u24a4 [PARENTHESIZED LATIN SMALL LETTER I]
+"\u24A4" => "(i)"
+
+# \u0133 [LATIN SMALL LIGATURE IJ]
+"\u0133" => "ij"
+
+# \u0134 [LATIN CAPITAL LETTER J WITH CIRCUMFLEX]
+"\u0134" => "J"
+
+# \u0248 [LATIN CAPITAL LETTER J WITH STROKE]
+"\u0248" => "J"
+
+# \u1d0a [LATIN LETTER SMALL CAPITAL J]
+"\u1D0A" => "J"
+
+# \u24bf [CIRCLED LATIN CAPITAL LETTER J]
+"\u24BF" => "J"
+
+# \uff2a [FULLWIDTH LATIN CAPITAL LETTER J]
+"\uFF2A" => "J"
+
+# \u0135 [LATIN SMALL LETTER J WITH CIRCUMFLEX]
+"\u0135" => "j"
+
+# \u01f0 [LATIN SMALL LETTER J WITH CARON]
+"\u01F0" => "j"
+
+# \u0237 [LATIN SMALL LETTER DOTLESS J]
+"\u0237" => "j"
+
+# \u0249 [LATIN SMALL LETTER J WITH STROKE]
+"\u0249" => "j"
+
+# \u025f [LATIN SMALL LETTER DOTLESS J WITH STROKE]
+"\u025F" => "j"
+
+# \u0284 [LATIN SMALL LETTER DOTLESS J WITH STROKE AND HOOK]
+"\u0284" => "j"
+
+# \u029d [LATIN SMALL LETTER J WITH CROSSED-TAIL]
+"\u029D" => "j"
+
+# \u24d9 [CIRCLED LATIN SMALL LETTER J]
+"\u24D9" => "j"
+
+# \u2c7c [LATIN SUBSCRIPT SMALL LETTER J]
+"\u2C7C" => "j"
+
+# \uff4a [FULLWIDTH LATIN SMALL LETTER J]
+"\uFF4A" => "j"
+
+# \u24a5 [PARENTHESIZED LATIN SMALL LETTER J]
+"\u24A5" => "(j)"
+
+# \u0136 [LATIN CAPITAL LETTER K WITH CEDILLA]
+"\u0136" => "K"
+
+# \u0198 [LATIN CAPITAL LETTER K WITH HOOK]
+"\u0198" => "K"
+
+# \u01e8 [LATIN CAPITAL LETTER K WITH CARON]
+"\u01E8" => "K"
+
+# \u1d0b [LATIN LETTER SMALL CAPITAL K]
+"\u1D0B" => "K"
+
+# \u1e30 [LATIN CAPITAL LETTER K WITH ACUTE]
+"\u1E30" => "K"
+
+# \u1e32 [LATIN CAPITAL LETTER K WITH DOT BELOW]
+"\u1E32" => "K"
+
+# \u1e34 [LATIN CAPITAL LETTER K WITH LINE BELOW]
+"\u1E34" => "K"
+
+# \u24c0 [CIRCLED LATIN CAPITAL LETTER K]
+"\u24C0" => "K"
+
+# \u2c69 [LATIN CAPITAL LETTER K WITH DESCENDER]
+"\u2C69" => "K"
+
+# \ua740 [LATIN CAPITAL LETTER K WITH STROKE]
+"\uA740" => "K"
+
+# \ua742 [LATIN CAPITAL LETTER K WITH DIAGONAL STROKE]
+"\uA742" => "K"
+
+# \ua744 [LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE]
+"\uA744" => "K"
+
+# \uff2b [FULLWIDTH LATIN CAPITAL LETTER K]
+"\uFF2B" => "K"
+
+# \u0137 [LATIN SMALL LETTER K WITH CEDILLA]
+"\u0137" => "k"
+
+# \u0199 [LATIN SMALL LETTER K WITH HOOK]
+"\u0199" => "k"
+
+# \u01e9 [LATIN SMALL LETTER K WITH CARON]
+"\u01E9" => "k"
+
+# \u029e [LATIN SMALL LETTER TURNED K]
+"\u029E" => "k"
+
+# \u1d84 [LATIN SMALL LETTER K WITH PALATAL HOOK]
+"\u1D84" => "k"
+
+# \u1e31 [LATIN SMALL LETTER K WITH ACUTE]
+"\u1E31" => "k"
+
+# \u1e33 [LATIN SMALL LETTER K WITH DOT BELOW]
+"\u1E33" => "k"
+
+# \u1e35 [LATIN SMALL LETTER K WITH LINE BELOW]
+"\u1E35" => "k"
+
+# \u24da [CIRCLED LATIN SMALL LETTER K]
+"\u24DA" => "k"
+
+# \u2c6a [LATIN SMALL LETTER K WITH DESCENDER]
+"\u2C6A" => "k"
+
+# \ua741 [LATIN SMALL LETTER K WITH STROKE]
+"\uA741" => "k"
+
+# \ua743 [LATIN SMALL LETTER K WITH DIAGONAL STROKE]
+"\uA743" => "k"
+
+# \ua745 [LATIN SMALL LETTER K WITH STROKE AND DIAGONAL STROKE]
+"\uA745" => "k"
+
+# \uff4b [FULLWIDTH LATIN SMALL LETTER K]
+"\uFF4B" => "k"
+
+# \u24a6 [PARENTHESIZED LATIN SMALL LETTER K]
+"\u24A6" => "(k)"
+
+# \u0139 [LATIN CAPITAL LETTER L WITH ACUTE]
+"\u0139" => "L"
+
+# \u013b [LATIN CAPITAL LETTER L WITH CEDILLA]
+"\u013B" => "L"
+
+# \u013d [LATIN CAPITAL LETTER L WITH CARON]
+"\u013D" => "L"
+
+# \u013f [LATIN CAPITAL LETTER L WITH MIDDLE DOT]
+"\u013F" => "L"
+
+# \u0141 [LATIN CAPITAL LETTER L WITH STROKE]
+"\u0141" => "L"
+
+# \u023d [LATIN CAPITAL LETTER L WITH BAR]
+"\u023D" => "L"
+
+# \u029f [LATIN LETTER SMALL CAPITAL L]
+"\u029F" => "L"
+
+# \u1d0c [LATIN LETTER SMALL CAPITAL L WITH STROKE]
+"\u1D0C" => "L"
+
+# \u1e36 [LATIN CAPITAL LETTER L WITH DOT BELOW]
+"\u1E36" => "L"
+
+# \u1e38 [LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON]
+"\u1E38" => "L"
+
+# \u1e3a [LATIN CAPITAL LETTER L WITH LINE BELOW]
+"\u1E3A" => "L"
+
+# \u1e3c [LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW]
+"\u1E3C" => "L"
+
+# \u24c1 [CIRCLED LATIN CAPITAL LETTER L]
+"\u24C1" => "L"
+
+# \u2c60 [LATIN CAPITAL LETTER L WITH DOUBLE BAR]
+"\u2C60" => "L"
+
+# \u2c62 [LATIN CAPITAL LETTER L WITH MIDDLE TILDE]
+"\u2C62" => "L"
+
+# \ua746 [LATIN CAPITAL LETTER BROKEN L]
+"\uA746" => "L"
+
+# \ua748 [LATIN CAPITAL LETTER L WITH HIGH STROKE]
+"\uA748" => "L"
+
+# \ua780 [LATIN CAPITAL LETTER TURNED L]
+"\uA780" => "L"
+
+# \uff2c [FULLWIDTH LATIN CAPITAL LETTER L]
+"\uFF2C" => "L"
+
+# \u013a [LATIN SMALL LETTER L WITH ACUTE]
+"\u013A" => "l"
+
+# \u013c [LATIN SMALL LETTER L WITH CEDILLA]
+"\u013C" => "l"
+
+# \u013e [LATIN SMALL LETTER L WITH CARON]
+"\u013E" => "l"
+
+# \u0140 [LATIN SMALL LETTER L WITH MIDDLE DOT]
+"\u0140" => "l"
+
+# \u0142 [LATIN SMALL LETTER L WITH STROKE]
+"\u0142" => "l"
+
+# \u019a [LATIN SMALL LETTER L WITH BAR]
+"\u019A" => "l"
+
+# \u0234 [LATIN SMALL LETTER L WITH CURL]
+"\u0234" => "l"
+
+# \u026b [LATIN SMALL LETTER L WITH MIDDLE TILDE]
+"\u026B" => "l"
+
+# \u026c [LATIN SMALL LETTER L WITH BELT]
+"\u026C" => "l"
+
+# \u026d [LATIN SMALL LETTER L WITH RETROFLEX HOOK]
+"\u026D" => "l"
+
+# \u1d85 [LATIN SMALL LETTER L WITH PALATAL HOOK]
+"\u1D85" => "l"
+
+# \u1e37 [LATIN SMALL LETTER L WITH DOT BELOW]
+"\u1E37" => "l"
+
+# \u1e39 [LATIN SMALL LETTER L WITH DOT BELOW AND MACRON]
+"\u1E39" => "l"
+
+# \u1e3b [LATIN SMALL LETTER L WITH LINE BELOW]
+"\u1E3B" => "l"
+
+# \u1e3d [LATIN SMALL LETTER L WITH CIRCUMFLEX BELOW]
+"\u1E3D" => "l"
+
+# \u24db [CIRCLED LATIN SMALL LETTER L]
+"\u24DB" => "l"
+
+# \u2c61 [LATIN SMALL LETTER L WITH DOUBLE BAR]
+"\u2C61" => "l"
+
+# \ua747 [LATIN SMALL LETTER BROKEN L]
+"\uA747" => "l"
+
+# \ua749 [LATIN SMALL LETTER L WITH HIGH STROKE]
+"\uA749" => "l"
+
+# \ua781 [LATIN SMALL LETTER TURNED L]
+"\uA781" => "l"
+
+# \uff4c [FULLWIDTH LATIN SMALL LETTER L]
+"\uFF4C" => "l"
+
+# \u01c7 [LATIN CAPITAL LETTER LJ]
+"\u01C7" => "LJ"
+
+# \u1efa [LATIN CAPITAL LETTER MIDDLE-WELSH LL]
+"\u1EFA" => "LL"
+
+# \u01c8 [LATIN CAPITAL LETTER L WITH SMALL LETTER J]
+"\u01C8" => "Lj"
+
+# \u24a7 [PARENTHESIZED LATIN SMALL LETTER L]
+"\u24A7" => "(l)"
+
+# \u01c9 [LATIN SMALL LETTER LJ]
+"\u01C9" => "lj"
+
+# \u1efb [LATIN SMALL LETTER MIDDLE-WELSH LL]
+"\u1EFB" => "ll"
+
+# \u02aa [LATIN SMALL LETTER LS DIGRAPH]
+"\u02AA" => "ls"
+
+# \u02ab [LATIN SMALL LETTER LZ DIGRAPH]
+"\u02AB" => "lz"
+
+# \u019c [LATIN CAPITAL LETTER TURNED M]
+"\u019C" => "M"
+
+# \u1d0d [LATIN LETTER SMALL CAPITAL M]
+"\u1D0D" => "M"
+
+# \u1e3e [LATIN CAPITAL LETTER M WITH ACUTE]
+"\u1E3E" => "M"
+
+# \u1e40 [LATIN CAPITAL LETTER M WITH DOT ABOVE]
+"\u1E40" => "M"
+
+# \u1e42 [LATIN CAPITAL LETTER M WITH DOT BELOW]
+"\u1E42" => "M"
+
+# \u24c2 [CIRCLED LATIN CAPITAL LETTER M]
+"\u24C2" => "M"
+
+# \u2c6e [LATIN CAPITAL LETTER M WITH HOOK]
+"\u2C6E" => "M"
+
+# \ua7fd [LATIN EPIGRAPHIC LETTER INVERTED M]
+"\uA7FD" => "M"
+
+# \ua7ff [LATIN EPIGRAPHIC LETTER ARCHAIC M]
+"\uA7FF" => "M"
+
+# \uff2d [FULLWIDTH LATIN CAPITAL LETTER M]
+"\uFF2D" => "M"
+
+# \u026f [LATIN SMALL LETTER TURNED M]
+"\u026F" => "m"
+
+# \u0270 [LATIN SMALL LETTER TURNED M WITH LONG LEG]
+"\u0270" => "m"
+
+# \u0271 [LATIN SMALL LETTER M WITH HOOK]
+"\u0271" => "m"
+
+# \u1d6f [LATIN SMALL LETTER M WITH MIDDLE TILDE]
+"\u1D6F" => "m"
+
+# \u1d86 [LATIN SMALL LETTER M WITH PALATAL HOOK]
+"\u1D86" => "m"
+
+# \u1e3f [LATIN SMALL LETTER M WITH ACUTE]
+"\u1E3F" => "m"
+
+# \u1e41 [LATIN SMALL LETTER M WITH DOT ABOVE]
+"\u1E41" => "m"
+
+# \u1e43 [LATIN SMALL LETTER M WITH DOT BELOW]
+"\u1E43" => "m"
+
+# \u24dc [CIRCLED LATIN SMALL LETTER M]
+"\u24DC" => "m"
+
+# \uff4d [FULLWIDTH LATIN SMALL LETTER M]
+"\uFF4D" => "m"
+
+# \u24a8 [PARENTHESIZED LATIN SMALL LETTER M]
+"\u24A8" => "(m)"
+
+# � [LATIN CAPITAL LETTER N WITH TILDE]
+"\u00D1" => "N"
+
+# \u0143 [LATIN CAPITAL LETTER N WITH ACUTE]
+"\u0143" => "N"
+
+# \u0145 [LATIN CAPITAL LETTER N WITH CEDILLA]
+"\u0145" => "N"
+
+# \u0147 [LATIN CAPITAL LETTER N WITH CARON]
+"\u0147" => "N"
+
+# \u014a http://en.wikipedia.org/wiki/Eng_(letter) [LATIN CAPITAL LETTER ENG]
+"\u014A" => "N"
+
+# \u019d [LATIN CAPITAL LETTER N WITH LEFT HOOK]
+"\u019D" => "N"
+
+# \u01f8 [LATIN CAPITAL LETTER N WITH GRAVE]
+"\u01F8" => "N"
+
+# \u0220 [LATIN CAPITAL LETTER N WITH LONG RIGHT LEG]
+"\u0220" => "N"
+
+# \u0274 [LATIN LETTER SMALL CAPITAL N]
+"\u0274" => "N"
+
+# \u1d0e [LATIN LETTER SMALL CAPITAL REVERSED N]
+"\u1D0E" => "N"
+
+# \u1e44 [LATIN CAPITAL LETTER N WITH DOT ABOVE]
+"\u1E44" => "N"
+
+# \u1e46 [LATIN CAPITAL LETTER N WITH DOT BELOW]
+"\u1E46" => "N"
+
+# \u1e48 [LATIN CAPITAL LETTER N WITH LINE BELOW]
+"\u1E48" => "N"
+
+# \u1e4a [LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW]
+"\u1E4A" => "N"
+
+# \u24c3 [CIRCLED LATIN CAPITAL LETTER N]
+"\u24C3" => "N"
+
+# \uff2e [FULLWIDTH LATIN CAPITAL LETTER N]
+"\uFF2E" => "N"
+
+# � [LATIN SMALL LETTER N WITH TILDE]
+"\u00F1" => "n"
+
+# \u0144 [LATIN SMALL LETTER N WITH ACUTE]
+"\u0144" => "n"
+
+# \u0146 [LATIN SMALL LETTER N WITH CEDILLA]
+"\u0146" => "n"
+
+# \u0148 [LATIN SMALL LETTER N WITH CARON]
+"\u0148" => "n"
+
+# \u0149 [LATIN SMALL LETTER N PRECEDED BY APOSTROPHE]
+"\u0149" => "n"
+
+# \u014b http://en.wikipedia.org/wiki/Eng_(letter) [LATIN SMALL LETTER ENG]
+"\u014B" => "n"
+
+# \u019e [LATIN SMALL LETTER N WITH LONG RIGHT LEG]
+"\u019E" => "n"
+
+# \u01f9 [LATIN SMALL LETTER N WITH GRAVE]
+"\u01F9" => "n"
+
+# \u0235 [LATIN SMALL LETTER N WITH CURL]
+"\u0235" => "n"
+
+# \u0272 [LATIN SMALL LETTER N WITH LEFT HOOK]
+"\u0272" => "n"
+
+# \u0273 [LATIN SMALL LETTER N WITH RETROFLEX HOOK]
+"\u0273" => "n"
+
+# \u1d70 [LATIN SMALL LETTER N WITH MIDDLE TILDE]
+"\u1D70" => "n"
+
+# \u1d87 [LATIN SMALL LETTER N WITH PALATAL HOOK]
+"\u1D87" => "n"
+
+# \u1e45 [LATIN SMALL LETTER N WITH DOT ABOVE]
+"\u1E45" => "n"
+
+# \u1e47 [LATIN SMALL LETTER N WITH DOT BELOW]
+"\u1E47" => "n"
+
+# \u1e49 [LATIN SMALL LETTER N WITH LINE BELOW]
+"\u1E49" => "n"
+
+# \u1e4b [LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW]
+"\u1E4B" => "n"
+
+# \u207f [SUPERSCRIPT LATIN SMALL LETTER N]
+"\u207F" => "n"
+
+# \u24dd [CIRCLED LATIN SMALL LETTER N]
+"\u24DD" => "n"
+
+# \uff4e [FULLWIDTH LATIN SMALL LETTER N]
+"\uFF4E" => "n"
+
+# \u01ca [LATIN CAPITAL LETTER NJ]
+"\u01CA" => "NJ"
+
+# \u01cb [LATIN CAPITAL LETTER N WITH SMALL LETTER J]
+"\u01CB" => "Nj"
+
+# \u24a9 [PARENTHESIZED LATIN SMALL LETTER N]
+"\u24A9" => "(n)"
+
+# \u01cc [LATIN SMALL LETTER NJ]
+"\u01CC" => "nj"
+
+# � [LATIN CAPITAL LETTER O WITH GRAVE]
+"\u00D2" => "O"
+
+# � [LATIN CAPITAL LETTER O WITH ACUTE]
+"\u00D3" => "O"
+
+# � [LATIN CAPITAL LETTER O WITH CIRCUMFLEX]
+"\u00D4" => "O"
+
+# � [LATIN CAPITAL LETTER O WITH TILDE]
+"\u00D5" => "O"
+
+# � [LATIN CAPITAL LETTER O WITH DIAERESIS]
+"\u00D6" => "O"
+
+# � [LATIN CAPITAL LETTER O WITH STROKE]
+"\u00D8" => "O"
+
+# \u014c [LATIN CAPITAL LETTER O WITH MACRON]
+"\u014C" => "O"
+
+# \u014e [LATIN CAPITAL LETTER O WITH BREVE]
+"\u014E" => "O"
+
+# \u0150 [LATIN CAPITAL LETTER O WITH DOUBLE ACUTE]
+"\u0150" => "O"
+
+# \u0186 [LATIN CAPITAL LETTER OPEN O]
+"\u0186" => "O"
+
+# \u019f [LATIN CAPITAL LETTER O WITH MIDDLE TILDE]
+"\u019F" => "O"
+
+# \u01a0 [LATIN CAPITAL LETTER O WITH HORN]
+"\u01A0" => "O"
+
+# \u01d1 [LATIN CAPITAL LETTER O WITH CARON]
+"\u01D1" => "O"
+
+# \u01ea [LATIN CAPITAL LETTER O WITH OGONEK]
+"\u01EA" => "O"
+
+# \u01ec [LATIN CAPITAL LETTER O WITH OGONEK AND MACRON]
+"\u01EC" => "O"
+
+# \u01fe [LATIN CAPITAL LETTER O WITH STROKE AND ACUTE]
+"\u01FE" => "O"
+
+# \u020c [LATIN CAPITAL LETTER O WITH DOUBLE GRAVE]
+"\u020C" => "O"
+
+# \u020e [LATIN CAPITAL LETTER O WITH INVERTED BREVE]
+"\u020E" => "O"
+
+# \u022a [LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON]
+"\u022A" => "O"
+
+# \u022c [LATIN CAPITAL LETTER O WITH TILDE AND MACRON]
+"\u022C" => "O"
+
+# \u022e [LATIN CAPITAL LETTER O WITH DOT ABOVE]
+"\u022E" => "O"
+
+# \u0230 [LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON]
+"\u0230" => "O"
+
+# \u1d0f [LATIN LETTER SMALL CAPITAL O]
+"\u1D0F" => "O"
+
+# \u1d10 [LATIN LETTER SMALL CAPITAL OPEN O]
+"\u1D10" => "O"
+
+# \u1e4c [LATIN CAPITAL LETTER O WITH TILDE AND ACUTE]
+"\u1E4C" => "O"
+
+# \u1e4e [LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS]
+"\u1E4E" => "O"
+
+# \u1e50 [LATIN CAPITAL LETTER O WITH MACRON AND GRAVE]
+"\u1E50" => "O"
+
+# \u1e52 [LATIN CAPITAL LETTER O WITH MACRON AND ACUTE]
+"\u1E52" => "O"
+
+# \u1ecc [LATIN CAPITAL LETTER O WITH DOT BELOW]
+"\u1ECC" => "O"
+
+# \u1ece [LATIN CAPITAL LETTER O WITH HOOK ABOVE]
+"\u1ECE" => "O"
+
+# \u1ed0 [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE]
+"\u1ED0" => "O"
+
+# \u1ed2 [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE]
+"\u1ED2" => "O"
+
+# \u1ed4 [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE]
+"\u1ED4" => "O"
+
+# \u1ed6 [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE]
+"\u1ED6" => "O"
+
+# \u1ed8 [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW]
+"\u1ED8" => "O"
+
+# \u1eda [LATIN CAPITAL LETTER O WITH HORN AND ACUTE]
+"\u1EDA" => "O"
+
+# \u1edc [LATIN CAPITAL LETTER O WITH HORN AND GRAVE]
+"\u1EDC" => "O"
+
+# \u1ede [LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE]
+"\u1EDE" => "O"
+
+# \u1ee0 [LATIN CAPITAL LETTER O WITH HORN AND TILDE]
+"\u1EE0" => "O"
+
+# \u1ee2 [LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW]
+"\u1EE2" => "O"
+
+# \u24c4 [CIRCLED LATIN CAPITAL LETTER O]
+"\u24C4" => "O"
+
+# \ua74a [LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY]
+"\uA74A" => "O"
+
+# \ua74c [LATIN CAPITAL LETTER O WITH LOOP]
+"\uA74C" => "O"
+
+# \uff2f [FULLWIDTH LATIN CAPITAL LETTER O]
+"\uFF2F" => "O"
+
+# � [LATIN SMALL LETTER O WITH GRAVE]
+"\u00F2" => "o"
+
+# � [LATIN SMALL LETTER O WITH ACUTE]
+"\u00F3" => "o"
+
+# � [LATIN SMALL LETTER O WITH CIRCUMFLEX]
+"\u00F4" => "o"
+
+# � [LATIN SMALL LETTER O WITH TILDE]
+"\u00F5" => "o"
+
+# � [LATIN SMALL LETTER O WITH DIAERESIS]
+"\u00F6" => "o"
+
+# � [LATIN SMALL LETTER O WITH STROKE]
+"\u00F8" => "o"
+
+# \u014d [LATIN SMALL LETTER O WITH MACRON]
+"\u014D" => "o"
+
+# \u014f [LATIN SMALL LETTER O WITH BREVE]
+"\u014F" => "o"
+
+# \u0151 [LATIN SMALL LETTER O WITH DOUBLE ACUTE]
+"\u0151" => "o"
+
+# \u01a1 [LATIN SMALL LETTER O WITH HORN]
+"\u01A1" => "o"
+
+# \u01d2 [LATIN SMALL LETTER O WITH CARON]
+"\u01D2" => "o"
+
+# \u01eb [LATIN SMALL LETTER O WITH OGONEK]
+"\u01EB" => "o"
+
+# \u01ed [LATIN SMALL LETTER O WITH OGONEK AND MACRON]
+"\u01ED" => "o"
+
+# \u01ff [LATIN SMALL LETTER O WITH STROKE AND ACUTE]
+"\u01FF" => "o"
+
+# \u020d [LATIN SMALL LETTER O WITH DOUBLE GRAVE]
+"\u020D" => "o"
+
+# \u020f [LATIN SMALL LETTER O WITH INVERTED BREVE]
+"\u020F" => "o"
+
+# \u022b [LATIN SMALL LETTER O WITH DIAERESIS AND MACRON]
+"\u022B" => "o"
+
+# \u022d [LATIN SMALL LETTER O WITH TILDE AND MACRON]
+"\u022D" => "o"
+
+# \u022f [LATIN SMALL LETTER O WITH DOT ABOVE]
+"\u022F" => "o"
+
+# \u0231 [LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON]
+"\u0231" => "o"
+
+# \u0254 [LATIN SMALL LETTER OPEN O]
+"\u0254" => "o"
+
+# \u0275 [LATIN SMALL LETTER BARRED O]
+"\u0275" => "o"
+
+# \u1d16 [LATIN SMALL LETTER TOP HALF O]
+"\u1D16" => "o"
+
+# \u1d17 [LATIN SMALL LETTER BOTTOM HALF O]
+"\u1D17" => "o"
+
+# \u1d97 [LATIN SMALL LETTER OPEN O WITH RETROFLEX HOOK]
+"\u1D97" => "o"
+
+# \u1e4d [LATIN SMALL LETTER O WITH TILDE AND ACUTE]
+"\u1E4D" => "o"
+
+# \u1e4f [LATIN SMALL LETTER O WITH TILDE AND DIAERESIS]
+"\u1E4F" => "o"
+
+# \u1e51 [LATIN SMALL LETTER O WITH MACRON AND GRAVE]
+"\u1E51" => "o"
+
+# \u1e53 [LATIN SMALL LETTER O WITH MACRON AND ACUTE]
+"\u1E53" => "o"
+
+# \u1ecd [LATIN SMALL LETTER O WITH DOT BELOW]
+"\u1ECD" => "o"
+
+# \u1ecf [LATIN SMALL LETTER O WITH HOOK ABOVE]
+"\u1ECF" => "o"
+
+# \u1ed1 [LATIN SMALL LETTER O WITH CIRCUMFLEX AND ACUTE]
+"\u1ED1" => "o"
+
+# \u1ed3 [LATIN SMALL LETTER O WITH CIRCUMFLEX AND GRAVE]
+"\u1ED3" => "o"
+
+# \u1ed5 [LATIN SMALL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE]
+"\u1ED5" => "o"
+
+# \u1ed7 [LATIN SMALL LETTER O WITH CIRCUMFLEX AND TILDE]
+"\u1ED7" => "o"
+
+# \u1ed9 [LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW]
+"\u1ED9" => "o"
+
+# \u1edb [LATIN SMALL LETTER O WITH HORN AND ACUTE]
+"\u1EDB" => "o"
+
+# \u1edd [LATIN SMALL LETTER O WITH HORN AND GRAVE]
+"\u1EDD" => "o"
+
+# \u1edf [LATIN SMALL LETTER O WITH HORN AND HOOK ABOVE]
+"\u1EDF" => "o"
+
+# \u1ee1 [LATIN SMALL LETTER O WITH HORN AND TILDE]
+"\u1EE1" => "o"
+
+# \u1ee3 [LATIN SMALL LETTER O WITH HORN AND DOT BELOW]
+"\u1EE3" => "o"
+
+# \u2092 [LATIN SUBSCRIPT SMALL LETTER O]
+"\u2092" => "o"
+
+# \u24de [CIRCLED LATIN SMALL LETTER O]
+"\u24DE" => "o"
+
+# \u2c7a [LATIN SMALL LETTER O WITH LOW RING INSIDE]
+"\u2C7A" => "o"
+
+# \ua74b [LATIN SMALL LETTER O WITH LONG STROKE OVERLAY]
+"\uA74B" => "o"
+
+# \ua74d [LATIN SMALL LETTER O WITH LOOP]
+"\uA74D" => "o"
+
+# \uff4f [FULLWIDTH LATIN SMALL LETTER O]
+"\uFF4F" => "o"
+
+# \u0152 [LATIN CAPITAL LIGATURE OE]
+"\u0152" => "OE"
+
+# \u0276 [LATIN LETTER SMALL CAPITAL OE]
+"\u0276" => "OE"
+
+# \ua74e [LATIN CAPITAL LETTER OO]
+"\uA74E" => "OO"
+
+# \u0222 http://en.wikipedia.org/wiki/OU [LATIN CAPITAL LETTER OU]
+"\u0222" => "OU"
+
+# \u1d15 [LATIN LETTER SMALL CAPITAL OU]
+"\u1D15" => "OU"
+
+# \u24aa [PARENTHESIZED LATIN SMALL LETTER O]
+"\u24AA" => "(o)"
+
+# \u0153 [LATIN SMALL LIGATURE OE]
+"\u0153" => "oe"
+
+# \u1d14 [LATIN SMALL LETTER TURNED OE]
+"\u1D14" => "oe"
+
+# \ua74f [LATIN SMALL LETTER OO]
+"\uA74F" => "oo"
+
+# \u0223 http://en.wikipedia.org/wiki/OU [LATIN SMALL LETTER OU]
+"\u0223" => "ou"
+
+# \u01a4 [LATIN CAPITAL LETTER P WITH HOOK]
+"\u01A4" => "P"
+
+# \u1d18 [LATIN LETTER SMALL CAPITAL P]
+"\u1D18" => "P"
+
+# \u1e54 [LATIN CAPITAL LETTER P WITH ACUTE]
+"\u1E54" => "P"
+
+# \u1e56 [LATIN CAPITAL LETTER P WITH DOT ABOVE]
+"\u1E56" => "P"
+
+# \u24c5 [CIRCLED LATIN CAPITAL LETTER P]
+"\u24C5" => "P"
+
+# \u2c63 [LATIN CAPITAL LETTER P WITH STROKE]
+"\u2C63" => "P"
+
+# \ua750 [LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER]
+"\uA750" => "P"
+
+# \ua752 [LATIN CAPITAL LETTER P WITH FLOURISH]
+"\uA752" => "P"
+
+# \ua754 [LATIN CAPITAL LETTER P WITH SQUIRREL TAIL]
+"\uA754" => "P"
+
+# \uff30 [FULLWIDTH LATIN CAPITAL LETTER P]
+"\uFF30" => "P"
+
+# \u01a5 [LATIN SMALL LETTER P WITH HOOK]
+"\u01A5" => "p"
+
+# \u1d71 [LATIN SMALL LETTER P WITH MIDDLE TILDE]
+"\u1D71" => "p"
+
+# \u1d7d [LATIN SMALL LETTER P WITH STROKE]
+"\u1D7D" => "p"
+
+# \u1d88 [LATIN SMALL LETTER P WITH PALATAL HOOK]
+"\u1D88" => "p"
+
+# \u1e55 [LATIN SMALL LETTER P WITH ACUTE]
+"\u1E55" => "p"
+
+# \u1e57 [LATIN SMALL LETTER P WITH DOT ABOVE]
+"\u1E57" => "p"
+
+# \u24df [CIRCLED LATIN SMALL LETTER P]
+"\u24DF" => "p"
+
+# \ua751 [LATIN SMALL LETTER P WITH STROKE THROUGH DESCENDER]
+"\uA751" => "p"
+
+# \ua753 [LATIN SMALL LETTER P WITH FLOURISH]
+"\uA753" => "p"
+
+# \ua755 [LATIN SMALL LETTER P WITH SQUIRREL TAIL]
+"\uA755" => "p"
+
+# \ua7fc [LATIN EPIGRAPHIC LETTER REVERSED P]
+"\uA7FC" => "p"
+
+# \uff50 [FULLWIDTH LATIN SMALL LETTER P]
+"\uFF50" => "p"
+
+# \u24ab [PARENTHESIZED LATIN SMALL LETTER P]
+"\u24AB" => "(p)"
+
+# \u024a [LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL]
+"\u024A" => "Q"
+
+# \u24c6 [CIRCLED LATIN CAPITAL LETTER Q]
+"\u24C6" => "Q"
+
+# \ua756 [LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER]
+"\uA756" => "Q"
+
+# \ua758 [LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE]
+"\uA758" => "Q"
+
+# \uff31 [FULLWIDTH LATIN CAPITAL LETTER Q]
+"\uFF31" => "Q"
+
+# \u0138 http://en.wikipedia.org/wiki/Kra_(letter) [LATIN SMALL LETTER KRA]
+"\u0138" => "q"
+
+# \u024b [LATIN SMALL LETTER Q WITH HOOK TAIL]
+"\u024B" => "q"
+
+# \u02a0 [LATIN SMALL LETTER Q WITH HOOK]
+"\u02A0" => "q"
+
+# \u24e0 [CIRCLED LATIN SMALL LETTER Q]
+"\u24E0" => "q"
+
+# \ua757 [LATIN SMALL LETTER Q WITH STROKE THROUGH DESCENDER]
+"\uA757" => "q"
+
+# \ua759 [LATIN SMALL LETTER Q WITH DIAGONAL STROKE]
+"\uA759" => "q"
+
+# \uff51 [FULLWIDTH LATIN SMALL LETTER Q]
+"\uFF51" => "q"
+
+# \u24ac [PARENTHESIZED LATIN SMALL LETTER Q]
+"\u24AC" => "(q)"
+
+# \u0239 [LATIN SMALL LETTER QP DIGRAPH]
+"\u0239" => "qp"
+
+# \u0154 [LATIN CAPITAL LETTER R WITH ACUTE]
+"\u0154" => "R"
+
+# \u0156 [LATIN CAPITAL LETTER R WITH CEDILLA]
+"\u0156" => "R"
+
+# \u0158 [LATIN CAPITAL LETTER R WITH CARON]
+"\u0158" => "R"
+
+# \u0212 [LATIN CAPITAL LETTER R WITH DOUBLE GRAVE]
+"\u0210" => "R"
+
+# \u0212 [LATIN CAPITAL LETTER R WITH INVERTED BREVE]
+"\u0212" => "R"
+
+# \u024c [LATIN CAPITAL LETTER R WITH STROKE]
+"\u024C" => "R"
+
+# \u0280 [LATIN LETTER SMALL CAPITAL R]
+"\u0280" => "R"
+
+# \u0281 [LATIN LETTER SMALL CAPITAL INVERTED R]
+"\u0281" => "R"
+
+# \u1d19 [LATIN LETTER SMALL CAPITAL REVERSED R]
+"\u1D19" => "R"
+
+# \u1d1a [LATIN LETTER SMALL CAPITAL TURNED R]
+"\u1D1A" => "R"
+
+# \u1e58 [LATIN CAPITAL LETTER R WITH DOT ABOVE]
+"\u1E58" => "R"
+
+# \u1e5a [LATIN CAPITAL LETTER R WITH DOT BELOW]
+"\u1E5A" => "R"
+
+# \u1e5c [LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON]
+"\u1E5C" => "R"
+
+# \u1e5e [LATIN CAPITAL LETTER R WITH LINE BELOW]
+"\u1E5E" => "R"
+
+# \u24c7 [CIRCLED LATIN CAPITAL LETTER R]
+"\u24C7" => "R"
+
+# \u2c64 [LATIN CAPITAL LETTER R WITH TAIL]
+"\u2C64" => "R"
+
+# \ua75a [LATIN CAPITAL LETTER R ROTUNDA]
+"\uA75A" => "R"
+
+# \ua782 [LATIN CAPITAL LETTER INSULAR R]
+"\uA782" => "R"
+
+# \uff32 [FULLWIDTH LATIN CAPITAL LETTER R]
+"\uFF32" => "R"
+
+# \u0155 [LATIN SMALL LETTER R WITH ACUTE]
+"\u0155" => "r"
+
+# \u0157 [LATIN SMALL LETTER R WITH CEDILLA]
+"\u0157" => "r"
+
+# \u0159 [LATIN SMALL LETTER R WITH CARON]
+"\u0159" => "r"
+
+# \u0211 [LATIN SMALL LETTER R WITH DOUBLE GRAVE]
+"\u0211" => "r"
+
+# \u0213 [LATIN SMALL LETTER R WITH INVERTED BREVE]
+"\u0213" => "r"
+
+# \u024d [LATIN SMALL LETTER R WITH STROKE]
+"\u024D" => "r"
+
+# \u027c [LATIN SMALL LETTER R WITH LONG LEG]
+"\u027C" => "r"
+
+# \u027d [LATIN SMALL LETTER R WITH TAIL]
+"\u027D" => "r"
+
+# \u027e [LATIN SMALL LETTER R WITH FISHHOOK]
+"\u027E" => "r"
+
+# \u027f [LATIN SMALL LETTER REVERSED R WITH FISHHOOK]
+"\u027F" => "r"
+
+# \u1d63 [LATIN SUBSCRIPT SMALL LETTER R]
+"\u1D63" => "r"
+
+# \u1d72 [LATIN SMALL LETTER R WITH MIDDLE TILDE]
+"\u1D72" => "r"
+
+# \u1d73 [LATIN SMALL LETTER R WITH FISHHOOK AND MIDDLE TILDE]
+"\u1D73" => "r"
+
+# \u1d89 [LATIN SMALL LETTER R WITH PALATAL HOOK]
+"\u1D89" => "r"
+
+# \u1e59 [LATIN SMALL LETTER R WITH DOT ABOVE]
+"\u1E59" => "r"
+
+# \u1e5b [LATIN SMALL LETTER R WITH DOT BELOW]
+"\u1E5B" => "r"
+
+# \u1e5d [LATIN SMALL LETTER R WITH DOT BELOW AND MACRON]
+"\u1E5D" => "r"
+
+# \u1e5f [LATIN SMALL LETTER R WITH LINE BELOW]
+"\u1E5F" => "r"
+
+# \u24e1 [CIRCLED LATIN SMALL LETTER R]
+"\u24E1" => "r"
+
+# \ua75b [LATIN SMALL LETTER R ROTUNDA]
+"\uA75B" => "r"
+
+# \ua783 [LATIN SMALL LETTER INSULAR R]
+"\uA783" => "r"
+
+# \uff52 [FULLWIDTH LATIN SMALL LETTER R]
+"\uFF52" => "r"
+
+# \u24ad [PARENTHESIZED LATIN SMALL LETTER R]
+"\u24AD" => "(r)"
+
+# \u015a [LATIN CAPITAL LETTER S WITH ACUTE]
+"\u015A" => "S"
+
+# \u015c [LATIN CAPITAL LETTER S WITH CIRCUMFLEX]
+"\u015C" => "S"
+
+# \u015e [LATIN CAPITAL LETTER S WITH CEDILLA]
+"\u015E" => "S"
+
+# \u0160 [LATIN CAPITAL LETTER S WITH CARON]
+"\u0160" => "S"
+
+# \u0218 [LATIN CAPITAL LETTER S WITH COMMA BELOW]
+"\u0218" => "S"
+
+# \u1e60 [LATIN CAPITAL LETTER S WITH DOT ABOVE]
+"\u1E60" => "S"
+
+# \u1e62 [LATIN CAPITAL LETTER S WITH DOT BELOW]
+"\u1E62" => "S"
+
+# \u1e64 [LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE]
+"\u1E64" => "S"
+
+# \u1e66 [LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE]
+"\u1E66" => "S"
+
+# \u1e68 [LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE]
+"\u1E68" => "S"
+
+# \u24c8 [CIRCLED LATIN CAPITAL LETTER S]
+"\u24C8" => "S"
+
+# \ua731 [LATIN LETTER SMALL CAPITAL S]
+"\uA731" => "S"
+
+# \ua785 [LATIN SMALL LETTER INSULAR S]
+"\uA785" => "S"
+
+# \uff33 [FULLWIDTH LATIN CAPITAL LETTER S]
+"\uFF33" => "S"
+
+# \u015b [LATIN SMALL LETTER S WITH ACUTE]
+"\u015B" => "s"
+
+# \u015d [LATIN SMALL LETTER S WITH CIRCUMFLEX]
+"\u015D" => "s"
+
+# \u015f [LATIN SMALL LETTER S WITH CEDILLA]
+"\u015F" => "s"
+
+# \u0161 [LATIN SMALL LETTER S WITH CARON]
+"\u0161" => "s"
+
+# \u017f http://en.wikipedia.org/wiki/Long_S [LATIN SMALL LETTER LONG S]
+"\u017F" => "s"
+
+# \u0219 [LATIN SMALL LETTER S WITH COMMA BELOW]
+"\u0219" => "s"
+
+# \u023f [LATIN SMALL LETTER S WITH SWASH TAIL]
+"\u023F" => "s"
+
+# \u0282 [LATIN SMALL LETTER S WITH HOOK]
+"\u0282" => "s"
+
+# \u1d74 [LATIN SMALL LETTER S WITH MIDDLE TILDE]
+"\u1D74" => "s"
+
+# \u1d8a [LATIN SMALL LETTER S WITH PALATAL HOOK]
+"\u1D8A" => "s"
+
+# \u1e61 [LATIN SMALL LETTER S WITH DOT ABOVE]
+"\u1E61" => "s"
+
+# \u1e63 [LATIN SMALL LETTER S WITH DOT BELOW]
+"\u1E63" => "s"
+
+# \u1e65 [LATIN SMALL LETTER S WITH ACUTE AND DOT ABOVE]
+"\u1E65" => "s"
+
+# \u1e67 [LATIN SMALL LETTER S WITH CARON AND DOT ABOVE]
+"\u1E67" => "s"
+
+# \u1e69 [LATIN SMALL LETTER S WITH DOT BELOW AND DOT ABOVE]
+"\u1E69" => "s"
+
+# \u1e9c [LATIN SMALL LETTER LONG S WITH DIAGONAL STROKE]
+"\u1E9C" => "s"
+
+# \u1e9d [LATIN SMALL LETTER LONG S WITH HIGH STROKE]
+"\u1E9D" => "s"
+
+# \u24e2 [CIRCLED LATIN SMALL LETTER S]
+"\u24E2" => "s"
+
+# \ua784 [LATIN CAPITAL LETTER INSULAR S]
+"\uA784" => "s"
+
+# \uff53 [FULLWIDTH LATIN SMALL LETTER S]
+"\uFF53" => "s"
+
+# \u1e9e [LATIN CAPITAL LETTER SHARP S]
+"\u1E9E" => "SS"
+
+# \u24ae [PARENTHESIZED LATIN SMALL LETTER S]
+"\u24AE" => "(s)"
+
+# � [LATIN SMALL LETTER SHARP S]
+"\u00DF" => "ss"
+
+# \ufb06 [LATIN SMALL LIGATURE ST]
+"\uFB06" => "st"
+
+# \u0162 [LATIN CAPITAL LETTER T WITH CEDILLA]
+"\u0162" => "T"
+
+# \u0164 [LATIN CAPITAL LETTER T WITH CARON]
+"\u0164" => "T"
+
+# \u0166 [LATIN CAPITAL LETTER T WITH STROKE]
+"\u0166" => "T"
+
+# \u01ac [LATIN CAPITAL LETTER T WITH HOOK]
+"\u01AC" => "T"
+
+# \u01ae [LATIN CAPITAL LETTER T WITH RETROFLEX HOOK]
+"\u01AE" => "T"
+
+# \u021a [LATIN CAPITAL LETTER T WITH COMMA BELOW]
+"\u021A" => "T"
+
+# \u023e [LATIN CAPITAL LETTER T WITH DIAGONAL STROKE]
+"\u023E" => "T"
+
+# \u1d1b [LATIN LETTER SMALL CAPITAL T]
+"\u1D1B" => "T"
+
+# \u1e6a [LATIN CAPITAL LETTER T WITH DOT ABOVE]
+"\u1E6A" => "T"
+
+# \u1e6c [LATIN CAPITAL LETTER T WITH DOT BELOW]
+"\u1E6C" => "T"
+
+# \u1e6e [LATIN CAPITAL LETTER T WITH LINE BELOW]
+"\u1E6E" => "T"
+
+# \u1e70 [LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW]
+"\u1E70" => "T"
+
+# \u24c9 [CIRCLED LATIN CAPITAL LETTER T]
+"\u24C9" => "T"
+
+# \ua786 [LATIN CAPITAL LETTER INSULAR T]
+"\uA786" => "T"
+
+# \uff34 [FULLWIDTH LATIN CAPITAL LETTER T]
+"\uFF34" => "T"
+
+# \u0163 [LATIN SMALL LETTER T WITH CEDILLA]
+"\u0163" => "t"
+
+# \u0165 [LATIN SMALL LETTER T WITH CARON]
+"\u0165" => "t"
+
+# \u0167 [LATIN SMALL LETTER T WITH STROKE]
+"\u0167" => "t"
+
+# \u01ab [LATIN SMALL LETTER T WITH PALATAL HOOK]
+"\u01AB" => "t"
+
+# \u01ad [LATIN SMALL LETTER T WITH HOOK]
+"\u01AD" => "t"
+
+# \u021b [LATIN SMALL LETTER T WITH COMMA BELOW]
+"\u021B" => "t"
+
+# \u0236 [LATIN SMALL LETTER T WITH CURL]
+"\u0236" => "t"
+
+# \u0287 [LATIN SMALL LETTER TURNED T]
+"\u0287" => "t"
+
+# \u0288 [LATIN SMALL LETTER T WITH RETROFLEX HOOK]
+"\u0288" => "t"
+
+# \u1d75 [LATIN SMALL LETTER T WITH MIDDLE TILDE]
+"\u1D75" => "t"
+
+# \u1e6b [LATIN SMALL LETTER T WITH DOT ABOVE]
+"\u1E6B" => "t"
+
+# \u1e6d [LATIN SMALL LETTER T WITH DOT BELOW]
+"\u1E6D" => "t"
+
+# \u1e6f [LATIN SMALL LETTER T WITH LINE BELOW]
+"\u1E6F" => "t"
+
+# \u1e71 [LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW]
+"\u1E71" => "t"
+
+# \u1e97 [LATIN SMALL LETTER T WITH DIAERESIS]
+"\u1E97" => "t"
+
+# \u24e3 [CIRCLED LATIN SMALL LETTER T]
+"\u24E3" => "t"
+
+# \u2c66 [LATIN SMALL LETTER T WITH DIAGONAL STROKE]
+"\u2C66" => "t"
+
+# \uff54 [FULLWIDTH LATIN SMALL LETTER T]
+"\uFF54" => "t"
+
+# � [LATIN CAPITAL LETTER THORN]
+"\u00DE" => "TH"
+
+# \ua766 [LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER]
+"\uA766" => "TH"
+
+# \ua728 [LATIN CAPITAL LETTER TZ]
+"\uA728" => "TZ"
+
+# \u24af [PARENTHESIZED LATIN SMALL LETTER T]
+"\u24AF" => "(t)"
+
+# \u02a8 [LATIN SMALL LETTER TC DIGRAPH WITH CURL]
+"\u02A8" => "tc"
+
+# � [LATIN SMALL LETTER THORN]
+"\u00FE" => "th"
+
+# \u1d7a [LATIN SMALL LETTER TH WITH STRIKETHROUGH]
+"\u1D7A" => "th"
+
+# \ua767 [LATIN SMALL LETTER THORN WITH STROKE THROUGH DESCENDER]
+"\uA767" => "th"
+
+# \u02a6 [LATIN SMALL LETTER TS DIGRAPH]
+"\u02A6" => "ts"
+
+# \ua729 [LATIN SMALL LETTER TZ]
+"\uA729" => "tz"
+
+# � [LATIN CAPITAL LETTER U WITH GRAVE]
+"\u00D9" => "U"
+
+# � [LATIN CAPITAL LETTER U WITH ACUTE]
+"\u00DA" => "U"
+
+# � [LATIN CAPITAL LETTER U WITH CIRCUMFLEX]
+"\u00DB" => "U"
+
+# � [LATIN CAPITAL LETTER U WITH DIAERESIS]
+"\u00DC" => "U"
+
+# \u0168 [LATIN CAPITAL LETTER U WITH TILDE]
+"\u0168" => "U"
+
+# \u016a [LATIN CAPITAL LETTER U WITH MACRON]
+"\u016A" => "U"
+
+# \u016c [LATIN CAPITAL LETTER U WITH BREVE]
+"\u016C" => "U"
+
+# \u016e [LATIN CAPITAL LETTER U WITH RING ABOVE]
+"\u016E" => "U"
+
+# \u0170 [LATIN CAPITAL LETTER U WITH DOUBLE ACUTE]
+"\u0170" => "U"
+
+# \u0172 [LATIN CAPITAL LETTER U WITH OGONEK]
+"\u0172" => "U"
+
+# \u01af [LATIN CAPITAL LETTER U WITH HORN]
+"\u01AF" => "U"
+
+# \u01d3 [LATIN CAPITAL LETTER U WITH CARON]
+"\u01D3" => "U"
+
+# \u01d5 [LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON]
+"\u01D5" => "U"
+
+# \u01d7 [LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE]
+"\u01D7" => "U"
+
+# \u01d9 [LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON]
+"\u01D9" => "U"
+
+# \u01db [LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE]
+"\u01DB" => "U"
+
+# \u0214 [LATIN CAPITAL LETTER U WITH DOUBLE GRAVE]
+"\u0214" => "U"
+
+# \u0216 [LATIN CAPITAL LETTER U WITH INVERTED BREVE]
+"\u0216" => "U"
+
+# \u0244 [LATIN CAPITAL LETTER U BAR]
+"\u0244" => "U"
+
+# \u1d1c [LATIN LETTER SMALL CAPITAL U]
+"\u1D1C" => "U"
+
+# \u1d7e [LATIN SMALL CAPITAL LETTER U WITH STROKE]
+"\u1D7E" => "U"
+
+# \u1e72 [LATIN CAPITAL LETTER U WITH DIAERESIS BELOW]
+"\u1E72" => "U"
+
+# \u1e74 [LATIN CAPITAL LETTER U WITH TILDE BELOW]
+"\u1E74" => "U"
+
+# \u1e76 [LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW]
+"\u1E76" => "U"
+
+# \u1e78 [LATIN CAPITAL LETTER U WITH TILDE AND ACUTE]
+"\u1E78" => "U"
+
+# \u1e7a [LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS]
+"\u1E7A" => "U"
+
+# \u1ee4 [LATIN CAPITAL LETTER U WITH DOT BELOW]
+"\u1EE4" => "U"
+
+# \u1ee6 [LATIN CAPITAL LETTER U WITH HOOK ABOVE]
+"\u1EE6" => "U"
+
+# \u1ee8 [LATIN CAPITAL LETTER U WITH HORN AND ACUTE]
+"\u1EE8" => "U"
+
+# \u1eea [LATIN CAPITAL LETTER U WITH HORN AND GRAVE]
+"\u1EEA" => "U"
+
+# \u1eec [LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE]
+"\u1EEC" => "U"
+
+# \u1eee [LATIN CAPITAL LETTER U WITH HORN AND TILDE]
+"\u1EEE" => "U"
+
+# \u1ef0 [LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW]
+"\u1EF0" => "U"
+
+# \u24ca [CIRCLED LATIN CAPITAL LETTER U]
+"\u24CA" => "U"
+
+# \uff35 [FULLWIDTH LATIN CAPITAL LETTER U]
+"\uFF35" => "U"
+
+# � [LATIN SMALL LETTER U WITH GRAVE]
+"\u00F9" => "u"
+
+# � [LATIN SMALL LETTER U WITH ACUTE]
+"\u00FA" => "u"
+
+# � [LATIN SMALL LETTER U WITH CIRCUMFLEX]
+"\u00FB" => "u"
+
+# � [LATIN SMALL LETTER U WITH DIAERESIS]
+"\u00FC" => "u"
+
+# \u0169 [LATIN SMALL LETTER U WITH TILDE]
+"\u0169" => "u"
+
+# \u016b [LATIN SMALL LETTER U WITH MACRON]
+"\u016B" => "u"
+
+# \u016d [LATIN SMALL LETTER U WITH BREVE]
+"\u016D" => "u"
+
+# \u016f [LATIN SMALL LETTER U WITH RING ABOVE]
+"\u016F" => "u"
+
+# \u0171 [LATIN SMALL LETTER U WITH DOUBLE ACUTE]
+"\u0171" => "u"
+
+# \u0173 [LATIN SMALL LETTER U WITH OGONEK]
+"\u0173" => "u"
+
+# \u01b0 [LATIN SMALL LETTER U WITH HORN]
+"\u01B0" => "u"
+
+# \u01d4 [LATIN SMALL LETTER U WITH CARON]
+"\u01D4" => "u"
+
+# \u01d6 [LATIN SMALL LETTER U WITH DIAERESIS AND MACRON]
+"\u01D6" => "u"
+
+# \u01d8 [LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE]
+"\u01D8" => "u"
+
+# \u01da [LATIN SMALL LETTER U WITH DIAERESIS AND CARON]
+"\u01DA" => "u"
+
+# \u01dc [LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE]
+"\u01DC" => "u"
+
+# \u0215 [LATIN SMALL LETTER U WITH DOUBLE GRAVE]
+"\u0215" => "u"
+
+# \u0217 [LATIN SMALL LETTER U WITH INVERTED BREVE]
+"\u0217" => "u"
+
+# \u0289 [LATIN SMALL LETTER U BAR]
+"\u0289" => "u"
+
+# \u1d64 [LATIN SUBSCRIPT SMALL LETTER U]
+"\u1D64" => "u"
+
+# \u1d99 [LATIN SMALL LETTER U WITH RETROFLEX HOOK]
+"\u1D99" => "u"
+
+# \u1e73 [LATIN SMALL LETTER U WITH DIAERESIS BELOW]
+"\u1E73" => "u"
+
+# \u1e75 [LATIN SMALL LETTER U WITH TILDE BELOW]
+"\u1E75" => "u"
+
+# \u1e77 [LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW]
+"\u1E77" => "u"
+
+# \u1e79 [LATIN SMALL LETTER U WITH TILDE AND ACUTE]
+"\u1E79" => "u"
+
+# \u1e7b [LATIN SMALL LETTER U WITH MACRON AND DIAERESIS]
+"\u1E7B" => "u"
+
+# \u1ee5 [LATIN SMALL LETTER U WITH DOT BELOW]
+"\u1EE5" => "u"
+
+# \u1ee7 [LATIN SMALL LETTER U WITH HOOK ABOVE]
+"\u1EE7" => "u"
+
+# \u1ee9 [LATIN SMALL LETTER U WITH HORN AND ACUTE]
+"\u1EE9" => "u"
+
+# \u1eeb [LATIN SMALL LETTER U WITH HORN AND GRAVE]
+"\u1EEB" => "u"
+
+# \u1eed [LATIN SMALL LETTER U WITH HORN AND HOOK ABOVE]
+"\u1EED" => "u"
+
+# \u1eef [LATIN SMALL LETTER U WITH HORN AND TILDE]
+"\u1EEF" => "u"
+
+# \u1ef1 [LATIN SMALL LETTER U WITH HORN AND DOT BELOW]
+"\u1EF1" => "u"
+
+# \u24e4 [CIRCLED LATIN SMALL LETTER U]
+"\u24E4" => "u"
+
+# \uff55 [FULLWIDTH LATIN SMALL LETTER U]
+"\uFF55" => "u"
+
+# \u24b0 [PARENTHESIZED LATIN SMALL LETTER U]
+"\u24B0" => "(u)"
+
+# \u1d6b [LATIN SMALL LETTER UE]
+"\u1D6B" => "ue"
+
+# \u01b2 [LATIN CAPITAL LETTER V WITH HOOK]
+"\u01B2" => "V"
+
+# \u0245 [LATIN CAPITAL LETTER TURNED V]
+"\u0245" => "V"
+
+# \u1d20 [LATIN LETTER SMALL CAPITAL V]
+"\u1D20" => "V"
+
+# \u1e7c [LATIN CAPITAL LETTER V WITH TILDE]
+"\u1E7C" => "V"
+
+# \u1e7e [LATIN CAPITAL LETTER V WITH DOT BELOW]
+"\u1E7E" => "V"
+
+# \u1efc [LATIN CAPITAL LETTER MIDDLE-WELSH V]
+"\u1EFC" => "V"
+
+# \u24cb [CIRCLED LATIN CAPITAL LETTER V]
+"\u24CB" => "V"
+
+# \ua75e [LATIN CAPITAL LETTER V WITH DIAGONAL STROKE]
+"\uA75E" => "V"
+
+# \ua768 [LATIN CAPITAL LETTER VEND]
+"\uA768" => "V"
+
+# \uff36 [FULLWIDTH LATIN CAPITAL LETTER V]
+"\uFF36" => "V"
+
+# \u028b [LATIN SMALL LETTER V WITH HOOK]
+"\u028B" => "v"
+
+# \u028c [LATIN SMALL LETTER TURNED V]
+"\u028C" => "v"
+
+# \u1d65 [LATIN SUBSCRIPT SMALL LETTER V]
+"\u1D65" => "v"
+
+# \u1d8c [LATIN SMALL LETTER V WITH PALATAL HOOK]
+"\u1D8C" => "v"
+
+# \u1e7d [LATIN SMALL LETTER V WITH TILDE]
+"\u1E7D" => "v"
+
+# \u1e7f [LATIN SMALL LETTER V WITH DOT BELOW]
+"\u1E7F" => "v"
+
+# \u24e5 [CIRCLED LATIN SMALL LETTER V]
+"\u24E5" => "v"
+
+# \u2c71 [LATIN SMALL LETTER V WITH RIGHT HOOK]
+"\u2C71" => "v"
+
+# \u2c74 [LATIN SMALL LETTER V WITH CURL]
+"\u2C74" => "v"
+
+# \ua75f [LATIN SMALL LETTER V WITH DIAGONAL STROKE]
+"\uA75F" => "v"
+
+# \uff56 [FULLWIDTH LATIN SMALL LETTER V]
+"\uFF56" => "v"
+
+# \ua760 [LATIN CAPITAL LETTER VY]
+"\uA760" => "VY"
+
+# \u24b1 [PARENTHESIZED LATIN SMALL LETTER V]
+"\u24B1" => "(v)"
+
+# \ua761 [LATIN SMALL LETTER VY]
+"\uA761" => "vy"
+
+# \u0174 [LATIN CAPITAL LETTER W WITH CIRCUMFLEX]
+"\u0174" => "W"
+
+# \u01f7 http://en.wikipedia.org/wiki/Wynn [LATIN CAPITAL LETTER WYNN]
+"\u01F7" => "W"
+
+# \u1d21 [LATIN LETTER SMALL CAPITAL W]
+"\u1D21" => "W"
+
+# \u1e80 [LATIN CAPITAL LETTER W WITH GRAVE]
+"\u1E80" => "W"
+
+# \u1e82 [LATIN CAPITAL LETTER W WITH ACUTE]
+"\u1E82" => "W"
+
+# \u1e84 [LATIN CAPITAL LETTER W WITH DIAERESIS]
+"\u1E84" => "W"
+
+# \u1e86 [LATIN CAPITAL LETTER W WITH DOT ABOVE]
+"\u1E86" => "W"
+
+# \u1e88 [LATIN CAPITAL LETTER W WITH DOT BELOW]
+"\u1E88" => "W"
+
+# \u24cc [CIRCLED LATIN CAPITAL LETTER W]
+"\u24CC" => "W"
+
+# \u2c72 [LATIN CAPITAL LETTER W WITH HOOK]
+"\u2C72" => "W"
+
+# \uff37 [FULLWIDTH LATIN CAPITAL LETTER W]
+"\uFF37" => "W"
+
+# \u0175 [LATIN SMALL LETTER W WITH CIRCUMFLEX]
+"\u0175" => "w"
+
+# \u01bf http://en.wikipedia.org/wiki/Wynn [LATIN LETTER WYNN]
+"\u01BF" => "w"
+
+# \u028d [LATIN SMALL LETTER TURNED W]
+"\u028D" => "w"
+
+# \u1e81 [LATIN SMALL LETTER W WITH GRAVE]
+"\u1E81" => "w"
+
+# \u1e83 [LATIN SMALL LETTER W WITH ACUTE]
+"\u1E83" => "w"
+
+# \u1e85 [LATIN SMALL LETTER W WITH DIAERESIS]
+"\u1E85" => "w"
+
+# \u1e87 [LATIN SMALL LETTER W WITH DOT ABOVE]
+"\u1E87" => "w"
+
+# \u1e89 [LATIN SMALL LETTER W WITH DOT BELOW]
+"\u1E89" => "w"
+
+# \u1e98 [LATIN SMALL LETTER W WITH RING ABOVE]
+"\u1E98" => "w"
+
+# \u24e6 [CIRCLED LATIN SMALL LETTER W]
+"\u24E6" => "w"
+
+# \u2c73 [LATIN SMALL LETTER W WITH HOOK]
+"\u2C73" => "w"
+
+# \uff57 [FULLWIDTH LATIN SMALL LETTER W]
+"\uFF57" => "w"
+
+# \u24b2 [PARENTHESIZED LATIN SMALL LETTER W]
+"\u24B2" => "(w)"
+
+# \u1e8a [LATIN CAPITAL LETTER X WITH DOT ABOVE]
+"\u1E8A" => "X"
+
+# \u1e8c [LATIN CAPITAL LETTER X WITH DIAERESIS]
+"\u1E8C" => "X"
+
+# \u24cd [CIRCLED LATIN CAPITAL LETTER X]
+"\u24CD" => "X"
+
+# \uff38 [FULLWIDTH LATIN CAPITAL LETTER X]
+"\uFF38" => "X"
+
+# \u1d8d [LATIN SMALL LETTER X WITH PALATAL HOOK]
+"\u1D8D" => "x"
+
+# \u1e8b [LATIN SMALL LETTER X WITH DOT ABOVE]
+"\u1E8B" => "x"
+
+# \u1e8d [LATIN SMALL LETTER X WITH DIAERESIS]
+"\u1E8D" => "x"
+
+# \u2093 [LATIN SUBSCRIPT SMALL LETTER X]
+"\u2093" => "x"
+
+# \u24e7 [CIRCLED LATIN SMALL LETTER X]
+"\u24E7" => "x"
+
+# \uff58 [FULLWIDTH LATIN SMALL LETTER X]
+"\uFF58" => "x"
+
+# \u24b3 [PARENTHESIZED LATIN SMALL LETTER X]
+"\u24B3" => "(x)"
+
+# � [LATIN CAPITAL LETTER Y WITH ACUTE]
+"\u00DD" => "Y"
+
+# \u0176 [LATIN CAPITAL LETTER Y WITH CIRCUMFLEX]
+"\u0176" => "Y"
+
+# \u0178 [LATIN CAPITAL LETTER Y WITH DIAERESIS]
+"\u0178" => "Y"
+
+# \u01b3 [LATIN CAPITAL LETTER Y WITH HOOK]
+"\u01B3" => "Y"
+
+# \u0232 [LATIN CAPITAL LETTER Y WITH MACRON]
+"\u0232" => "Y"
+
+# \u024e [LATIN CAPITAL LETTER Y WITH STROKE]
+"\u024E" => "Y"
+
+# \u028f [LATIN LETTER SMALL CAPITAL Y]
+"\u028F" => "Y"
+
+# \u1e8e [LATIN CAPITAL LETTER Y WITH DOT ABOVE]
+"\u1E8E" => "Y"
+
+# \u1ef2 [LATIN CAPITAL LETTER Y WITH GRAVE]
+"\u1EF2" => "Y"
+
+# \u1ef4 [LATIN CAPITAL LETTER Y WITH DOT BELOW]
+"\u1EF4" => "Y"
+
+# \u1ef6 [LATIN CAPITAL LETTER Y WITH HOOK ABOVE]
+"\u1EF6" => "Y"
+
+# \u1ef8 [LATIN CAPITAL LETTER Y WITH TILDE]
+"\u1EF8" => "Y"
+
+# \u1efe [LATIN CAPITAL LETTER Y WITH LOOP]
+"\u1EFE" => "Y"
+
+# \u24ce [CIRCLED LATIN CAPITAL LETTER Y]
+"\u24CE" => "Y"
+
+# \uff39 [FULLWIDTH LATIN CAPITAL LETTER Y]
+"\uFF39" => "Y"
+
+# � [LATIN SMALL LETTER Y WITH ACUTE]
+"\u00FD" => "y"
+
+# � [LATIN SMALL LETTER Y WITH DIAERESIS]
+"\u00FF" => "y"
+
+# \u0177 [LATIN SMALL LETTER Y WITH CIRCUMFLEX]
+"\u0177" => "y"
+
+# \u01b4 [LATIN SMALL LETTER Y WITH HOOK]
+"\u01B4" => "y"
+
+# \u0233 [LATIN SMALL LETTER Y WITH MACRON]
+"\u0233" => "y"
+
+# \u024f [LATIN SMALL LETTER Y WITH STROKE]
+"\u024F" => "y"
+
+# \u028e [LATIN SMALL LETTER TURNED Y]
+"\u028E" => "y"
+
+# \u1e8f [LATIN SMALL LETTER Y WITH DOT ABOVE]
+"\u1E8F" => "y"
+
+# \u1e99 [LATIN SMALL LETTER Y WITH RING ABOVE]
+"\u1E99" => "y"
+
+# \u1ef3 [LATIN SMALL LETTER Y WITH GRAVE]
+"\u1EF3" => "y"
+
+# \u1ef5 [LATIN SMALL LETTER Y WITH DOT BELOW]
+"\u1EF5" => "y"
+
+# \u1ef7 [LATIN SMALL LETTER Y WITH HOOK ABOVE]
+"\u1EF7" => "y"
+
+# \u1ef9 [LATIN SMALL LETTER Y WITH TILDE]
+"\u1EF9" => "y"
+
+# \u1eff [LATIN SMALL LETTER Y WITH LOOP]
+"\u1EFF" => "y"
+
+# \u24e8 [CIRCLED LATIN SMALL LETTER Y]
+"\u24E8" => "y"
+
+# \uff59 [FULLWIDTH LATIN SMALL LETTER Y]
+"\uFF59" => "y"
+
+# \u24b4 [PARENTHESIZED LATIN SMALL LETTER Y]
+"\u24B4" => "(y)"
+
+# \u0179 [LATIN CAPITAL LETTER Z WITH ACUTE]
+"\u0179" => "Z"
+
+# \u017b [LATIN CAPITAL LETTER Z WITH DOT ABOVE]
+"\u017B" => "Z"
+
+# \u017d [LATIN CAPITAL LETTER Z WITH CARON]
+"\u017D" => "Z"
+
+# \u01b5 [LATIN CAPITAL LETTER Z WITH STROKE]
+"\u01B5" => "Z"
+
+# \u021c http://en.wikipedia.org/wiki/Yogh [LATIN CAPITAL LETTER YOGH]
+"\u021C" => "Z"
+
+# \u0224 [LATIN CAPITAL LETTER Z WITH HOOK]
+"\u0224" => "Z"
+
+# \u1d22 [LATIN LETTER SMALL CAPITAL Z]
+"\u1D22" => "Z"
+
+# \u1e90 [LATIN CAPITAL LETTER Z WITH CIRCUMFLEX]
+"\u1E90" => "Z"
+
+# \u1e92 [LATIN CAPITAL LETTER Z WITH DOT BELOW]
+"\u1E92" => "Z"
+
+# \u1e94 [LATIN CAPITAL LETTER Z WITH LINE BELOW]
+"\u1E94" => "Z"
+
+# \u24cf [CIRCLED LATIN CAPITAL LETTER Z]
+"\u24CF" => "Z"
+
+# \u2c6b [LATIN CAPITAL LETTER Z WITH DESCENDER]
+"\u2C6B" => "Z"
+
+# \ua762 [LATIN CAPITAL LETTER VISIGOTHIC Z]
+"\uA762" => "Z"
+
+# \uff3a [FULLWIDTH LATIN CAPITAL LETTER Z]
+"\uFF3A" => "Z"
+
+# \u017a [LATIN SMALL LETTER Z WITH ACUTE]
+"\u017A" => "z"
+
+# \u017c [LATIN SMALL LETTER Z WITH DOT ABOVE]
+"\u017C" => "z"
+
+# \u017e [LATIN SMALL LETTER Z WITH CARON]
+"\u017E" => "z"
+
+# \u01b6 [LATIN SMALL LETTER Z WITH STROKE]
+"\u01B6" => "z"
+
+# \u021d http://en.wikipedia.org/wiki/Yogh [LATIN SMALL LETTER YOGH]
+"\u021D" => "z"
+
+# \u0225 [LATIN SMALL LETTER Z WITH HOOK]
+"\u0225" => "z"
+
+# \u0240 [LATIN SMALL LETTER Z WITH SWASH TAIL]
+"\u0240" => "z"
+
+# \u0290 [LATIN SMALL LETTER Z WITH RETROFLEX HOOK]
+"\u0290" => "z"
+
+# \u0291 [LATIN SMALL LETTER Z WITH CURL]
+"\u0291" => "z"
+
+# \u1d76 [LATIN SMALL LETTER Z WITH MIDDLE TILDE]
+"\u1D76" => "z"
+
+# \u1d8e [LATIN SMALL LETTER Z WITH PALATAL HOOK]
+"\u1D8E" => "z"
+
+# \u1e91 [LATIN SMALL LETTER Z WITH CIRCUMFLEX]
+"\u1E91" => "z"
+
+# \u1e93 [LATIN SMALL LETTER Z WITH DOT BELOW]
+"\u1E93" => "z"
+
+# \u1e95 [LATIN SMALL LETTER Z WITH LINE BELOW]
+"\u1E95" => "z"
+
+# \u24e9 [CIRCLED LATIN SMALL LETTER Z]
+"\u24E9" => "z"
+
+# \u2c6c [LATIN SMALL LETTER Z WITH DESCENDER]
+"\u2C6C" => "z"
+
+# \ua763 [LATIN SMALL LETTER VISIGOTHIC Z]
+"\uA763" => "z"
+
+# \uff5a [FULLWIDTH LATIN SMALL LETTER Z]
+"\uFF5A" => "z"
+
+# \u24b5 [PARENTHESIZED LATIN SMALL LETTER Z]
+"\u24B5" => "(z)"
+
+# \u2070 [SUPERSCRIPT ZERO]
+"\u2070" => "0"
+
+# \u2080 [SUBSCRIPT ZERO]
+"\u2080" => "0"
+
+# \u24ea [CIRCLED DIGIT ZERO]
+"\u24EA" => "0"
+
+# \u24ff [NEGATIVE CIRCLED DIGIT ZERO]
+"\u24FF" => "0"
+
+# \uff10 [FULLWIDTH DIGIT ZERO]
+"\uFF10" => "0"
+
+# � [SUPERSCRIPT ONE]
+"\u00B9" => "1"
+
+# \u2081 [SUBSCRIPT ONE]
+"\u2081" => "1"
+
+# \u2460 [CIRCLED DIGIT ONE]
+"\u2460" => "1"
+
+# \u24f5 [DOUBLE CIRCLED DIGIT ONE]
+"\u24F5" => "1"
+
+# \u2776 [DINGBAT NEGATIVE CIRCLED DIGIT ONE]
+"\u2776" => "1"
+
+# \u2780 [DINGBAT CIRCLED SANS-SERIF DIGIT ONE]
+"\u2780" => "1"
+
+# \u278a [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT ONE]
+"\u278A" => "1"
+
+# \uff11 [FULLWIDTH DIGIT ONE]
+"\uFF11" => "1"
+
+# \u2488 [DIGIT ONE FULL STOP]
+"\u2488" => "1."
+
+# \u2474 [PARENTHESIZED DIGIT ONE]
+"\u2474" => "(1)"
+
+# � [SUPERSCRIPT TWO]
+"\u00B2" => "2"
+
+# \u2082 [SUBSCRIPT TWO]
+"\u2082" => "2"
+
+# \u2461 [CIRCLED DIGIT TWO]
+"\u2461" => "2"
+
+# \u24f6 [DOUBLE CIRCLED DIGIT TWO]
+"\u24F6" => "2"
+
+# \u2777 [DINGBAT NEGATIVE CIRCLED DIGIT TWO]
+"\u2777" => "2"
+
+# \u2781 [DINGBAT CIRCLED SANS-SERIF DIGIT TWO]
+"\u2781" => "2"
+
+# \u278b [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT TWO]
+"\u278B" => "2"
+
+# \uff12 [FULLWIDTH DIGIT TWO]
+"\uFF12" => "2"
+
+# \u2489 [DIGIT TWO FULL STOP]
+"\u2489" => "2."
+
+# \u2475 [PARENTHESIZED DIGIT TWO]
+"\u2475" => "(2)"
+
+# � [SUPERSCRIPT THREE]
+"\u00B3" => "3"
+
+# \u2083 [SUBSCRIPT THREE]
+"\u2083" => "3"
+
+# \u2462 [CIRCLED DIGIT THREE]
+"\u2462" => "3"
+
+# \u24f7 [DOUBLE CIRCLED DIGIT THREE]
+"\u24F7" => "3"
+
+# \u2778 [DINGBAT NEGATIVE CIRCLED DIGIT THREE]
+"\u2778" => "3"
+
+# \u2782 [DINGBAT CIRCLED SANS-SERIF DIGIT THREE]
+"\u2782" => "3"
+
+# \u278c [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT THREE]
+"\u278C" => "3"
+
+# \uff13 [FULLWIDTH DIGIT THREE]
+"\uFF13" => "3"
+
+# \u248a [DIGIT THREE FULL STOP]
+"\u248A" => "3."
+
+# \u2476 [PARENTHESIZED DIGIT THREE]
+"\u2476" => "(3)"
+
+# \u2074 [SUPERSCRIPT FOUR]
+"\u2074" => "4"
+
+# \u2084 [SUBSCRIPT FOUR]
+"\u2084" => "4"
+
+# \u2463 [CIRCLED DIGIT FOUR]
+"\u2463" => "4"
+
+# \u24f8 [DOUBLE CIRCLED DIGIT FOUR]
+"\u24F8" => "4"
+
+# \u2779 [DINGBAT NEGATIVE CIRCLED DIGIT FOUR]
+"\u2779" => "4"
+
+# \u2783 [DINGBAT CIRCLED SANS-SERIF DIGIT FOUR]
+"\u2783" => "4"
+
+# \u278d [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT FOUR]
+"\u278D" => "4"
+
+# \uff14 [FULLWIDTH DIGIT FOUR]
+"\uFF14" => "4"
+
+# \u248b [DIGIT FOUR FULL STOP]
+"\u248B" => "4."
+
+# \u2477 [PARENTHESIZED DIGIT FOUR]
+"\u2477" => "(4)"
+
+# \u2075 [SUPERSCRIPT FIVE]
+"\u2075" => "5"
+
+# \u2085 [SUBSCRIPT FIVE]
+"\u2085" => "5"
+
+# \u2464 [CIRCLED DIGIT FIVE]
+"\u2464" => "5"
+
+# \u24f9 [DOUBLE CIRCLED DIGIT FIVE]
+"\u24F9" => "5"
+
+# \u277a [DINGBAT NEGATIVE CIRCLED DIGIT FIVE]
+"\u277A" => "5"
+
+# \u2784 [DINGBAT CIRCLED SANS-SERIF DIGIT FIVE]
+"\u2784" => "5"
+
+# \u278e [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT FIVE]
+"\u278E" => "5"
+
+# \uff15 [FULLWIDTH DIGIT FIVE]
+"\uFF15" => "5"
+
+# \u248c [DIGIT FIVE FULL STOP]
+"\u248C" => "5."
+
+# \u2478 [PARENTHESIZED DIGIT FIVE]
+"\u2478" => "(5)"
+
+# \u2076 [SUPERSCRIPT SIX]
+"\u2076" => "6"
+
+# \u2086 [SUBSCRIPT SIX]
+"\u2086" => "6"
+
+# \u2465 [CIRCLED DIGIT SIX]
+"\u2465" => "6"
+
+# \u24fa [DOUBLE CIRCLED DIGIT SIX]
+"\u24FA" => "6"
+
+# \u277b [DINGBAT NEGATIVE CIRCLED DIGIT SIX]
+"\u277B" => "6"
+
+# \u2785 [DINGBAT CIRCLED SANS-SERIF DIGIT SIX]
+"\u2785" => "6"
+
+# \u278f [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT SIX]
+"\u278F" => "6"
+
+# \uff16 [FULLWIDTH DIGIT SIX]
+"\uFF16" => "6"
+
+# \u248d [DIGIT SIX FULL STOP]
+"\u248D" => "6."
+
+# \u2479 [PARENTHESIZED DIGIT SIX]
+"\u2479" => "(6)"
+
+# \u2077 [SUPERSCRIPT SEVEN]
+"\u2077" => "7"
+
+# \u2087 [SUBSCRIPT SEVEN]
+"\u2087" => "7"
+
+# \u2466 [CIRCLED DIGIT SEVEN]
+"\u2466" => "7"
+
+# \u24fb [DOUBLE CIRCLED DIGIT SEVEN]
+"\u24FB" => "7"
+
+# \u277c [DINGBAT NEGATIVE CIRCLED DIGIT SEVEN]
+"\u277C" => "7"
+
+# \u2786 [DINGBAT CIRCLED SANS-SERIF DIGIT SEVEN]
+"\u2786" => "7"
+
+# \u2790 [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT SEVEN]
+"\u2790" => "7"
+
+# \uff17 [FULLWIDTH DIGIT SEVEN]
+"\uFF17" => "7"
+
+# \u248e [DIGIT SEVEN FULL STOP]
+"\u248E" => "7."
+
+# \u247a [PARENTHESIZED DIGIT SEVEN]
+"\u247A" => "(7)"
+
+# \u2078 [SUPERSCRIPT EIGHT]
+"\u2078" => "8"
+
+# \u2088 [SUBSCRIPT EIGHT]
+"\u2088" => "8"
+
+# \u2467 [CIRCLED DIGIT EIGHT]
+"\u2467" => "8"
+
+# \u24fc [DOUBLE CIRCLED DIGIT EIGHT]
+"\u24FC" => "8"
+
+# \u277d [DINGBAT NEGATIVE CIRCLED DIGIT EIGHT]
+"\u277D" => "8"
+
+# \u2787 [DINGBAT CIRCLED SANS-SERIF DIGIT EIGHT]
+"\u2787" => "8"
+
+# \u2791 [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT EIGHT]
+"\u2791" => "8"
+
+# \uff18 [FULLWIDTH DIGIT EIGHT]
+"\uFF18" => "8"
+
+# \u248f [DIGIT EIGHT FULL STOP]
+"\u248F" => "8."
+
+# \u247b [PARENTHESIZED DIGIT EIGHT]
+"\u247B" => "(8)"
+
+# \u2079 [SUPERSCRIPT NINE]
+"\u2079" => "9"
+
+# \u2089 [SUBSCRIPT NINE]
+"\u2089" => "9"
+
+# \u2468 [CIRCLED DIGIT NINE]
+"\u2468" => "9"
+
+# \u24fd [DOUBLE CIRCLED DIGIT NINE]
+"\u24FD" => "9"
+
+# \u277e [DINGBAT NEGATIVE CIRCLED DIGIT NINE]
+"\u277E" => "9"
+
+# \u2788 [DINGBAT CIRCLED SANS-SERIF DIGIT NINE]
+"\u2788" => "9"
+
+# \u2792 [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT NINE]
+"\u2792" => "9"
+
+# \uff19 [FULLWIDTH DIGIT NINE]
+"\uFF19" => "9"
+
+# \u2490 [DIGIT NINE FULL STOP]
+"\u2490" => "9."
+
+# \u247c [PARENTHESIZED DIGIT NINE]
+"\u247C" => "(9)"
+
+# \u2469 [CIRCLED NUMBER TEN]
+"\u2469" => "10"
+
+# \u24fe [DOUBLE CIRCLED NUMBER TEN]
+"\u24FE" => "10"
+
+# \u277f [DINGBAT NEGATIVE CIRCLED NUMBER TEN]
+"\u277F" => "10"
+
+# \u2789 [DINGBAT CIRCLED SANS-SERIF NUMBER TEN]
+"\u2789" => "10"
+
+# \u2793 [DINGBAT NEGATIVE CIRCLED SANS-SERIF NUMBER TEN]
+"\u2793" => "10"
+
+# \u2491 [NUMBER TEN FULL STOP]
+"\u2491" => "10."
+
+# \u247d [PARENTHESIZED NUMBER TEN]
+"\u247D" => "(10)"
+
+# \u246a [CIRCLED NUMBER ELEVEN]
+"\u246A" => "11"
+
+# \u24eb [NEGATIVE CIRCLED NUMBER ELEVEN]
+"\u24EB" => "11"
+
+# \u2492 [NUMBER ELEVEN FULL STOP]
+"\u2492" => "11."
+
+# \u247e [PARENTHESIZED NUMBER ELEVEN]
+"\u247E" => "(11)"
+
+# \u246b [CIRCLED NUMBER TWELVE]
+"\u246B" => "12"
+
+# \u24ec [NEGATIVE CIRCLED NUMBER TWELVE]
+"\u24EC" => "12"
+
+# \u2493 [NUMBER TWELVE FULL STOP]
+"\u2493" => "12."
+
+# \u247f [PARENTHESIZED NUMBER TWELVE]
+"\u247F" => "(12)"
+
+# \u246c [CIRCLED NUMBER THIRTEEN]
+"\u246C" => "13"
+
+# \u24ed [NEGATIVE CIRCLED NUMBER THIRTEEN]
+"\u24ED" => "13"
+
+# \u2494 [NUMBER THIRTEEN FULL STOP]
+"\u2494" => "13."
+
+# \u2480 [PARENTHESIZED NUMBER THIRTEEN]
+"\u2480" => "(13)"
+
+# \u246d [CIRCLED NUMBER FOURTEEN]
+"\u246D" => "14"
+
+# \u24ee [NEGATIVE CIRCLED NUMBER FOURTEEN]
+"\u24EE" => "14"
+
+# \u2495 [NUMBER FOURTEEN FULL STOP]
+"\u2495" => "14."
+
+# \u2481 [PARENTHESIZED NUMBER FOURTEEN]
+"\u2481" => "(14)"
+
+# \u246e [CIRCLED NUMBER FIFTEEN]
+"\u246E" => "15"
+
+# \u24ef [NEGATIVE CIRCLED NUMBER FIFTEEN]
+"\u24EF" => "15"
+
+# \u2496 [NUMBER FIFTEEN FULL STOP]
+"\u2496" => "15."
+
+# \u2482 [PARENTHESIZED NUMBER FIFTEEN]
+"\u2482" => "(15)"
+
+# \u246f [CIRCLED NUMBER SIXTEEN]
+"\u246F" => "16"
+
+# \u24f0 [NEGATIVE CIRCLED NUMBER SIXTEEN]
+"\u24F0" => "16"
+
+# \u2497 [NUMBER SIXTEEN FULL STOP]
+"\u2497" => "16."
+
+# \u2483 [PARENTHESIZED NUMBER SIXTEEN]
+"\u2483" => "(16)"
+
+# \u2470 [CIRCLED NUMBER SEVENTEEN]
+"\u2470" => "17"
+
+# \u24f1 [NEGATIVE CIRCLED NUMBER SEVENTEEN]
+"\u24F1" => "17"
+
+# \u2498 [NUMBER SEVENTEEN FULL STOP]
+"\u2498" => "17."
+
+# \u2484 [PARENTHESIZED NUMBER SEVENTEEN]
+"\u2484" => "(17)"
+
+# \u2471 [CIRCLED NUMBER EIGHTEEN]
+"\u2471" => "18"
+
+# \u24f2 [NEGATIVE CIRCLED NUMBER EIGHTEEN]
+"\u24F2" => "18"
+
+# \u2499 [NUMBER EIGHTEEN FULL STOP]
+"\u2499" => "18."
+
+# \u2485 [PARENTHESIZED NUMBER EIGHTEEN]
+"\u2485" => "(18)"
+
+# \u2472 [CIRCLED NUMBER NINETEEN]
+"\u2472" => "19"
+
+# \u24f3 [NEGATIVE CIRCLED NUMBER NINETEEN]
+"\u24F3" => "19"
+
+# \u249a [NUMBER NINETEEN FULL STOP]
+"\u249A" => "19."
+
+# \u2486 [PARENTHESIZED NUMBER NINETEEN]
+"\u2486" => "(19)"
+
+# \u2473 [CIRCLED NUMBER TWENTY]
+"\u2473" => "20"
+
+# \u24f4 [NEGATIVE CIRCLED NUMBER TWENTY]
+"\u24F4" => "20"
+
+# \u249b [NUMBER TWENTY FULL STOP]
+"\u249B" => "20."
+
+# \u2487 [PARENTHESIZED NUMBER TWENTY]
+"\u2487" => "(20)"
+
+# � [LEFT-POINTING DOUBLE ANGLE QUOTATION MARK]
+"\u00AB" => "\""
+
+# � [RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK]
+"\u00BB" => "\""
+
+# \u201c [LEFT DOUBLE QUOTATION MARK]
+"\u201C" => "\""
+
+# \u201d [RIGHT DOUBLE QUOTATION MARK]
+"\u201D" => "\""
+
+# \u201e [DOUBLE LOW-9 QUOTATION MARK]
+"\u201E" => "\""
+
+# \u2033 [DOUBLE PRIME]
+"\u2033" => "\""
+
+# \u2036 [REVERSED DOUBLE PRIME]
+"\u2036" => "\""
+
+# \u275d [HEAVY DOUBLE TURNED COMMA QUOTATION MARK ORNAMENT]
+"\u275D" => "\""
+
+# \u275e [HEAVY DOUBLE COMMA QUOTATION MARK ORNAMENT]
+"\u275E" => "\""
+
+# \u276e [HEAVY LEFT-POINTING ANGLE QUOTATION MARK ORNAMENT]
+"\u276E" => "\""
+
+# \u276f [HEAVY RIGHT-POINTING ANGLE QUOTATION MARK ORNAMENT]
+"\u276F" => "\""
+
+# \uff02 [FULLWIDTH QUOTATION MARK]
+"\uFF02" => "\""
+
+# \u2018 [LEFT SINGLE QUOTATION MARK]
+"\u2018" => "\'"
+
+# \u2019 [RIGHT SINGLE QUOTATION MARK]
+"\u2019" => "\'"
+
+# \u201a [SINGLE LOW-9 QUOTATION MARK]
+"\u201A" => "\'"
+
+# \u201b [SINGLE HIGH-REVERSED-9 QUOTATION MARK]
+"\u201B" => "\'"
+
+# \u2032 [PRIME]
+"\u2032" => "\'"
+
+# \u2035 [REVERSED PRIME]
+"\u2035" => "\'"
+
+# \u2039 [SINGLE LEFT-POINTING ANGLE QUOTATION MARK]
+"\u2039" => "\'"
+
+# \u203a [SINGLE RIGHT-POINTING ANGLE QUOTATION MARK]
+"\u203A" => "\'"
+
+# \u275b [HEAVY SINGLE TURNED COMMA QUOTATION MARK ORNAMENT]
+"\u275B" => "\'"
+
+# \u275c [HEAVY SINGLE COMMA QUOTATION MARK ORNAMENT]
+"\u275C" => "\'"
+
+# \uff07 [FULLWIDTH APOSTROPHE]
+"\uFF07" => "\'"
+
+# \u2010 [HYPHEN]
+"\u2010" => "-"
+
+# \u2011 [NON-BREAKING HYPHEN]
+"\u2011" => "-"
+
+# \u2012 [FIGURE DASH]
+"\u2012" => "-"
+
+# \u2013 [EN DASH]
+"\u2013" => "-"
+
+# \u2014 [EM DASH]
+"\u2014" => "-"
+
+# \u207b [SUPERSCRIPT MINUS]
+"\u207B" => "-"
+
+# \u208b [SUBSCRIPT MINUS]
+"\u208B" => "-"
+
+# \uff0d [FULLWIDTH HYPHEN-MINUS]
+"\uFF0D" => "-"
+
+# \u2045 [LEFT SQUARE BRACKET WITH QUILL]
+"\u2045" => "["
+
+# \u2772 [LIGHT LEFT TORTOISE SHELL BRACKET ORNAMENT]
+"\u2772" => "["
+
+# \uff3b [FULLWIDTH LEFT SQUARE BRACKET]
+"\uFF3B" => "["
+
+# \u2046 [RIGHT SQUARE BRACKET WITH QUILL]
+"\u2046" => "]"
+
+# \u2773 [LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT]
+"\u2773" => "]"
+
+# \uff3d [FULLWIDTH RIGHT SQUARE BRACKET]
+"\uFF3D" => "]"
+
+# \u207d [SUPERSCRIPT LEFT PARENTHESIS]
+"\u207D" => "("
+
+# \u208d [SUBSCRIPT LEFT PARENTHESIS]
+"\u208D" => "("
+
+# \u2768 [MEDIUM LEFT PARENTHESIS ORNAMENT]
+"\u2768" => "("
+
+# \u276a [MEDIUM FLATTENED LEFT PARENTHESIS ORNAMENT]
+"\u276A" => "("
+
+# \uff08 [FULLWIDTH LEFT PARENTHESIS]
+"\uFF08" => "("
+
+# \u2e28 [LEFT DOUBLE PARENTHESIS]
+"\u2E28" => "(("
+
+# \u207e [SUPERSCRIPT RIGHT PARENTHESIS]
+"\u207E" => ")"
+
+# \u208e [SUBSCRIPT RIGHT PARENTHESIS]
+"\u208E" => ")"
+
+# \u2769 [MEDIUM RIGHT PARENTHESIS ORNAMENT]
+"\u2769" => ")"
+
+# \u276b [MEDIUM FLATTENED RIGHT PARENTHESIS ORNAMENT]
+"\u276B" => ")"
+
+# \uff09 [FULLWIDTH RIGHT PARENTHESIS]
+"\uFF09" => ")"
+
+# \u2e29 [RIGHT DOUBLE PARENTHESIS]
+"\u2E29" => "))"
+
+# \u276c [MEDIUM LEFT-POINTING ANGLE BRACKET ORNAMENT]
+"\u276C" => "<"
+
+# \u2770 [HEAVY LEFT-POINTING ANGLE BRACKET ORNAMENT]
+"\u2770" => "<"
+
+# \uff1c [FULLWIDTH LESS-THAN SIGN]
+"\uFF1C" => "<"
+
+# \u276d [MEDIUM RIGHT-POINTING ANGLE BRACKET ORNAMENT]
+"\u276D" => ">"
+
+# \u2771 [HEAVY RIGHT-POINTING ANGLE BRACKET ORNAMENT]
+"\u2771" => ">"
+
+# \uff1e [FULLWIDTH GREATER-THAN SIGN]
+"\uFF1E" => ">"
+
+# \u2774 [MEDIUM LEFT CURLY BRACKET ORNAMENT]
+"\u2774" => "{"
+
+# \uff5b [FULLWIDTH LEFT CURLY BRACKET]
+"\uFF5B" => "{"
+
+# \u2775 [MEDIUM RIGHT CURLY BRACKET ORNAMENT]
+"\u2775" => "}"
+
+# \uff5d [FULLWIDTH RIGHT CURLY BRACKET]
+"\uFF5D" => "}"
+
+# \u207a [SUPERSCRIPT PLUS SIGN]
+"\u207A" => "+"
+
+# \u208a [SUBSCRIPT PLUS SIGN]
+"\u208A" => "+"
+
+# \uff0b [FULLWIDTH PLUS SIGN]
+"\uFF0B" => "+"
+
+# \u207c [SUPERSCRIPT EQUALS SIGN]
+"\u207C" => "="
+
+# \u208c [SUBSCRIPT EQUALS SIGN]
+"\u208C" => "="
+
+# \uff1d [FULLWIDTH EQUALS SIGN]
+"\uFF1D" => "="
+
+# \uff01 [FULLWIDTH EXCLAMATION MARK]
+"\uFF01" => "!"
+
+# \u203c [DOUBLE EXCLAMATION MARK]
+"\u203C" => "!!"
+
+# \u2049 [EXCLAMATION QUESTION MARK]
+"\u2049" => "!?"
+
+# \uff03 [FULLWIDTH NUMBER SIGN]
+"\uFF03" => "#"
+
+# \uff04 [FULLWIDTH DOLLAR SIGN]
+"\uFF04" => "$"
+
+# \u2052 [COMMERCIAL MINUS SIGN]
+"\u2052" => "%"
+
+# \uff05 [FULLWIDTH PERCENT SIGN]
+"\uFF05" => "%"
+
+# \uff06 [FULLWIDTH AMPERSAND]
+"\uFF06" => "&"
+
+# \u204e [LOW ASTERISK]
+"\u204E" => "*"
+
+# \uff0a [FULLWIDTH ASTERISK]
+"\uFF0A" => "*"
+
+# \uff0c [FULLWIDTH COMMA]
+"\uFF0C" => ","
+
+# \uff0e [FULLWIDTH FULL STOP]
+"\uFF0E" => "."
+
+# \u2044 [FRACTION SLASH]
+"\u2044" => "/"
+
+# \uff0f [FULLWIDTH SOLIDUS]
+"\uFF0F" => "/"
+
+# \uff1a [FULLWIDTH COLON]
+"\uFF1A" => ":"
+
+# \u204f [REVERSED SEMICOLON]
+"\u204F" => ";"
+
+# \uff1b [FULLWIDTH SEMICOLON]
+"\uFF1B" => ";"
+
+# \uff1f [FULLWIDTH QUESTION MARK]
+"\uFF1F" => "?"
+
+# \u2047 [DOUBLE QUESTION MARK]
+"\u2047" => "??"
+
+# \u2048 [QUESTION EXCLAMATION MARK]
+"\u2048" => "?!"
+
+# \uff20 [FULLWIDTH COMMERCIAL AT]
+"\uFF20" => "@"
+
+# \uff3c [FULLWIDTH REVERSE SOLIDUS]
+"\uFF3C" => "\\"
+
+# \u2038 [CARET]
+"\u2038" => "^"
+
+# \uff3e [FULLWIDTH CIRCUMFLEX ACCENT]
+"\uFF3E" => "^"
+
+# \uff3f [FULLWIDTH LOW LINE]
+"\uFF3F" => "_"
+
+# \u2053 [SWUNG DASH]
+"\u2053" => "~"
+
+# \uff5e [FULLWIDTH TILDE]
+"\uFF5E" => "~"
+
+################################################################
+# Below is the Perl script used to generate the above mappings #
+# from ASCIIFoldingFilter.java: #
+################################################################
+#
+# #!/usr/bin/perl
+#
+# use warnings;
+# use strict;
+#
+# my @source_chars = ();
+# my @source_char_descriptions = ();
+# my $target = '';
+#
+# while (<>) {
+# if (/case\s+'(\\u[A-F0-9]+)':\s*\/\/\s*(.*)/i) {
+# push @source_chars, $1;
+# push @source_char_descriptions, $2;
+# next;
+# }
+# if (/output\[[^\]]+\]\s*=\s*'(\\'|\\\\|.)'/) {
+# $target .= $1;
+# next;
+# }
+# if (/break;/) {
+# $target = "\\\"" if ($target eq '"');
+# for my $source_char_num (0..$#source_chars) {
+# print "# $source_char_descriptions[$source_char_num]\n";
+# print "\"$source_chars[$source_char_num]\" => \"$target\"\n\n";
+# }
+# @source_chars = ();
+# @source_char_descriptions = ();
+# $target = '';
+# }
+# }
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ac2c9ba5/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
----------------------------------------------------------------------
diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
index 132c583..75c0a3b 100644
--- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
+++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
@@ -68,6 +68,17 @@ public interface PersistenceService {
boolean save(Item item);
/**
+ * Persists the specified Item in the context server.
+ *
+ * @param item the item to persist
+ * @param useBatching whether to use batching or not for saving the item. If activating there may be a delay between
+ * the call to this method and the actual saving in the persistence backend.
+ *
+ * @return {@code true} if the item was properly persisted, {@code false} otherwise
+ */
+ boolean save(Item item, boolean useBatching);
+
+ /**
* Updates the item of the specified class and identified by the specified identifier with new property values provided as name - value pairs in the specified Map.
*
* @param itemId the identifier of the item we want to update
[08/50] [abbrv] incubator-unomi git commit: UNOMI-63 Use
ElasticSearch BulkProcessing to perform segment updates - Track index
creation through internal memory structure
Posted by sh...@apache.org.
UNOMI-63 Use ElasticSearch BulkProcessing to perform segment updates
- Track index creation through internal memory structure
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/661daeea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/661daeea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/661daeea
Branch: refs/heads/master
Commit: 661daeea5951fe7219508e710609a965a3192989
Parents: 62d11de
Author: Serge Huber <sh...@apache.org>
Authored: Mon Nov 21 17:39:59 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Mon Nov 21 17:39:59 2016 +0100
----------------------------------------------------------------------
.../ElasticSearchPersistenceServiceImpl.java | 60 +++++++++++++++-----
1 file changed, 45 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/661daeea/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index b954b75..733b3a9 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -411,14 +411,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
bulkProcessor = getBulkProcessor();
}
- try {
- IndicesStatsResponse indicesStatsResponse = client.admin().indices().prepareStats().all().execute().get();
- existingIndexNames = new TreeSet<>(indicesStatsResponse.getIndices().keySet());
- } catch (InterruptedException e) {
- logger.error("Error retrieving indices stats", e);
- } catch (ExecutionException e) {
- logger.error("Error retrieving indices stats", e);
- }
+ refreshExistingIndexNames();
logger.info("Waiting for index creation to complete...");
@@ -457,7 +450,8 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
int thisMonth = gc.get(Calendar.MONTH);
gc.add(Calendar.DAY_OF_MONTH, 1);
if (gc.get(Calendar.MONTH) != thisMonth) {
- getMonthlyIndex(gc.getTime(), true);
+ String monthlyIndex = getMonthlyIndex(gc.getTime(), true);
+ existingIndexNames.add(monthlyIndex);
}
}
}, 10000L, 24L * 60L * 60L * 1000L);
@@ -465,6 +459,23 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
logger.info(this.getClass().getName() + " service started successfully.");
}
+ private void refreshExistingIndexNames() {
+ new InClassLoaderExecute<Boolean>() {
+ protected Boolean execute(Object... args) {
+ try {
+ logger.info("Refreshing existing indices list...");
+ IndicesStatsResponse indicesStatsResponse = client.admin().indices().prepareStats().all().execute().get();
+ existingIndexNames = new TreeSet<>(indicesStatsResponse.getIndices().keySet());
+ } catch (InterruptedException e) {
+ logger.error("Error retrieving indices stats", e);
+ } catch (ExecutionException e) {
+ logger.error("Error retrieving indices stats", e);
+ }
+ return true;
+ }
+ }.executeInClassLoader();
+ }
+
public BulkProcessor getBulkProcessor() {
if (bulkProcessor != null) {
return bulkProcessor;
@@ -490,7 +501,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
BulkRequest request,
Throwable failure) {
logger.error("After Bulk (failure)", failure);
- // we could add index creation here in the case of index seperation by dates.
}
});
if (bulkProcessorName != null && bulkProcessorName.length() > 0) {
@@ -554,7 +564,11 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
protected Object execute(Object... args) {
logger.info("Closing ElasticSearch persistence backend...");
if (bulkProcessor != null) {
- bulkProcessor.close();
+ try {
+ bulkProcessor.awaitClose(2, TimeUnit.MINUTES);
+ } catch (InterruptedException e) {
+ logger.error("Error waiting for bulk operations to flush !", e);
+ }
}
node.close();
return null;
@@ -730,17 +744,27 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
if (routingByType.containsKey(itemType)) {
indexBuilder = indexBuilder.setRouting(routingByType.get(itemType));
}
- try {
- indexBuilder.execute().actionGet();
- } catch (IndexNotFoundException e) {
+
+ if (!existingIndexNames.contains(index)) {
+ // index probably doesn't exist, unless something else has already created it.
if (itemsMonthlyIndexed.contains(itemType)) {
Date timeStamp = ((TimestampedItem) item).getTimeStamp();
if (timeStamp != null) {
getMonthlyIndex(timeStamp, true);
- indexBuilder.execute().actionGet();
} else {
logger.warn("Missing time stamp on item " + item + " id=" + item.getItemId() + " can't create related monthly index !");
}
+ } else {
+ // this is not a timestamped index, should we create it anyway ?
+ createIndex(index);
+ }
+ }
+
+ try {
+ indexBuilder.execute().actionGet();
+ } catch (IndexNotFoundException e) {
+ if (existingIndexNames.contains(index)) {
+ existingIndexNames.remove(index);
}
}
return true;
@@ -887,6 +911,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
boolean indexExists = indicesExistsResponse.isExists();
if (indexExists) {
client.admin().indices().prepareDelete(indexName).execute().actionGet();
+ existingIndexNames.remove(indexName);
}
return indexExists;
}
@@ -919,6 +944,8 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
builder.execute().actionGet();
+ existingIndexNames.add(indexName);
+
}
@@ -1417,6 +1444,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
public void refresh() {
new InClassLoaderExecute<Boolean>() {
protected Boolean execute(Object... args) {
+ if (bulkProcessor != null) {
+ bulkProcessor.flush();
+ }
client.admin().indices().refresh(Requests.refreshRequest()).actionGet();
return true;
}
[33/50] [abbrv] incubator-unomi git commit: Add toString method to
help debugging
Posted by sh...@apache.org.
Add toString method to help debugging
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/31677cbd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/31677cbd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/31677cbd
Branch: refs/heads/master
Commit: 31677cbd351cb0ca34249c0ad75adeace98461e6
Parents: ffe2591
Author: Serge Huber <sh...@apache.org>
Authored: Wed Dec 21 20:15:23 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Wed Dec 21 20:15:23 2016 +0100
----------------------------------------------------------------------
.../java/org/apache/unomi/api/conditions/Condition.java | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/31677cbd/api/src/main/java/org/apache/unomi/api/conditions/Condition.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/unomi/api/conditions/Condition.java b/api/src/main/java/org/apache/unomi/api/conditions/Condition.java
index 2165f0f..334c1ae 100644
--- a/api/src/main/java/org/apache/unomi/api/conditions/Condition.java
+++ b/api/src/main/java/org/apache/unomi/api/conditions/Condition.java
@@ -154,4 +154,14 @@ public class Condition {
result = 31 * result + parameterValues.hashCode();
return result;
}
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("Condition{");
+ sb.append("conditionType=").append(conditionType);
+ sb.append(", conditionTypeId='").append(conditionTypeId).append('\'');
+ sb.append(", parameterValues=").append(parameterValues);
+ sb.append('}');
+ return sb.toString();
+ }
}
[04/50] [abbrv] incubator-unomi git commit: Merge branch
'UNOMI-28-ES-2-X-UPGRADE' into feature-UNOMI-28-ES2X
Posted by sh...@apache.org.
Merge branch 'UNOMI-28-ES-2-X-UPGRADE' into feature-UNOMI-28-ES2X
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/73efe571
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/73efe571
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/73efe571
Branch: refs/heads/master
Commit: 73efe571cd8aa289dedc5cad1426c32b22e0c56e
Parents: 9493c70 ca1462d
Author: Serge Huber <sh...@apache.org>
Authored: Wed Nov 16 14:32:57 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Wed Nov 16 14:32:57 2016 +0100
----------------------------------------------------------------------
.../META-INF/cxs/mappings/geonameEntry.json | 2 +-
itests/pom.xml | 5 +-
.../java/org/apache/unomi/itests/AllTests.java | 11 +-
.../apache/unomi/itests/ProfileServiceTest.java | 53 +++
kar/src/main/feature/feature.xml | 12 +-
package/pom.xml | 53 ++-
persistence-elasticsearch/core/pom.xml | 202 +++-------
.../ElasticSearchPersistenceServiceImpl.java | 344 ++++++++++++++---
.../conditions/ConditionContextHelper.java | 4 +-
.../conditions/ConditionESQueryBuilder.java | 4 +-
.../ConditionESQueryBuilderDispatcher.java | 16 +-
.../META-INF/cxs/mappings/campaign.json | 4 +-
.../META-INF/cxs/mappings/campaignevent.json | 2 +-
.../resources/META-INF/cxs/mappings/event.json | 2 +-
.../META-INF/cxs/mappings/profile.json | 2 +-
.../META-INF/cxs/mappings/session.json | 4 +-
.../resources/OSGI-INF/blueprint/blueprint.xml | 13 +
.../core/src/main/resources/elasticsearch.yml | 381 +++----------------
...g.apache.unomi.persistence.elasticsearch.cfg | 6 +
.../plugin/security/SecurityPlugin.java | 14 +-
.../plugin/security/SecurityPluginModule.java | 3 +-
.../plugin/security/SecurityPluginService.java | 4 +-
.../BooleanConditionESQueryBuilder.java | 20 +-
...onByPointSessionConditionESQueryBuilder.java | 10 +-
.../MatchAllConditionESQueryBuilder.java | 8 +-
.../conditions/NotConditionESQueryBuilder.java | 8 +-
.../PastEventConditionESQueryBuilder.java | 8 +-
.../PropertyConditionESQueryBuilder.java | 51 +--
.../conditions/PropertyConditionEvaluator.java | 3 +-
...rceEventPropertyConditionESQueryBuilder.java | 27 +-
.../HoverEventConditionESQueryBuilder.java | 23 +-
pom.xml | 8 +-
.../services/services/SegmentServiceImpl.java | 8 +-
33 files changed, 629 insertions(+), 686 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/73efe571/itests/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/73efe571/package/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/73efe571/persistence-elasticsearch/core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/73efe571/pom.xml
----------------------------------------------------------------------
[32/50] [abbrv] incubator-unomi git commit: Update install
instructions for deployment using features
Posted by sh...@apache.org.
Update install instructions for deployment using features
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/ffe25912
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/ffe25912
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/ffe25912
Branch: refs/heads/master
Commit: ffe259125865aa15eaee8638951d9a1a7d8d0459
Parents: 2942393
Author: Serge Huber <sh...@apache.org>
Authored: Wed Dec 21 20:15:00 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Wed Dec 21 20:15:00 2016 +0100
----------------------------------------------------------------------
README.md | 15 +++++----------
1 file changed, 5 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ffe25912/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 7c807a7..b73efa4 100644
--- a/README.md
+++ b/README.md
@@ -75,9 +75,10 @@ on your disk and copy all the files from the lib/sigar directory into Karaf's li
3. Install the WAR support, CXF and CDI (OpenWebBeans) into Karaf by doing the following in the Karaf command line:
```
- feature:install -v war
- feature:repo-add cxf 2.7.11
- feature:install -v cxf/2.7.11
+ feature:repo-add cxf 3.0.2
+ feature:repo-add cellar 3.0.3
+ feature:repo-add mvn:org.apache.unomi/unomi-kar/VERSION/xml/features
+ feature:install unomi-kar
```
4. Create a new $MY_KARAF_HOME/etc/org.apache.cxf.osgi.cfg file and put the following property inside :
@@ -85,14 +86,8 @@ on your disk and copy all the files from the lib/sigar directory into Karaf's li
```
org.apache.cxf.servlet.context=/cxs
```
-
-5. Copy the following KAR to the Karaf deploy directory, as in this example line:
-
- ```
- cp kar/target/unomi-kar-1.0.0-SNAPSHOT.kar ~/java/deployments/unomi/apache-karaf-3.0.1/deploy/
- ```
-6. If all went smoothly, you should be able to access the context script here : http://localhost:8181/cxs/cluster .
+5. If all went smoothly, you should be able to access the context script here : http://localhost:8181/cxs/cluster .
You should be able to login with karaf / karaf and see basic server information. If not something went wrong during the install.
Configuration
[05/50] [abbrv] incubator-unomi git commit: UNOMI-63 Use
ElasticSearch BulkProcessing to perform segment updates - Add minimal
configuration documentation - Change bulk size value back to its default 5MB
size
Posted by sh...@apache.org.
UNOMI-63 Use ElasticSearch BulkProcessing to perform segment updates
- Add minimal configuration documentation
- Change bulk size value back to its default 5MB size
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/9a582147
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/9a582147
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/9a582147
Branch: refs/heads/master
Commit: 9a582147ef1bf2058d99172b392da5867ffbae52
Parents: 73efe57
Author: Serge Huber <sh...@apache.org>
Authored: Thu Nov 17 09:34:30 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Thu Nov 17 09:34:30 2016 +0100
----------------------------------------------------------------------
.../resources/org.apache.unomi.persistence.elasticsearch.cfg | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9a582147/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
index c5d906f..2b193f6 100644
--- a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
+++ b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
@@ -26,9 +26,13 @@ node.data=true
discovery.zen.ping.multicast.enabled=false
#discovery.zen.ping.unicast.hosts=["192.168.0.1:9300", "192.168.0.2:9300"]
defaultQueryLimit=10
+
+# The following settings control the behavior of the BulkProcessor API. You can find more information about these
+# settings and their behavior here : https://www.elastic.co/guide/en/elasticsearch/client/java-api/2.4/java-docs-bulk-processor.html
+# The values used here are the default values of the API
bulkProcessor.name=unomi-bulk
bulkProcessor.concurrentRequests=1
bulkProcessor.bulkActions=1000
-bulkProcessor.bulkSize=10MB
+bulkProcessor.bulkSize=5MB
bulkProcessor.flushInterval=5s
bulkProcessor.backoffPolicy=exponential
\ No newline at end of file
[12/50] [abbrv] incubator-unomi git commit: Merge PR #18
Posted by sh...@apache.org.
Merge PR #18
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/2263da96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/2263da96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/2263da96
Branch: refs/heads/master
Commit: 2263da963dfa04cba367a7bf68f889445d9b74b5
Parents: 2524693
Author: Serge Huber <sh...@apache.org>
Authored: Fri Nov 25 14:26:21 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Fri Nov 25 14:26:21 2016 +0100
----------------------------------------------------------------------
.../baseplugin/conditions/PropertyConditionESQueryBuilder.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/2263da96/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
index cf58734..7013581 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
@@ -111,14 +111,14 @@ public class PropertyConditionESQueryBuilder implements ConditionESQueryBuilder
return boolQueryBuilder;
case "hasSomeOf":
checkRequiredValue(values, name, op, true);
- BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+ boolQueryBuilder = QueryBuilders.boolQuery();
for (Object curValue : values) {
boolQueryBuilder.should(QueryBuilders.termQuery(name, curValue));
}
return boolQueryBuilder;
case "hasNoneOf":
checkRequiredValue(values, name, op, true);
- BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+ boolQueryBuilder = QueryBuilders.boolQuery();
for (Object curValue : values) {
boolQueryBuilder.mustNot(QueryBuilders.termQuery(name, curValue));
}
[27/50] [abbrv] incubator-unomi git commit: UNOMI-70 : use folding +
keyword mapping. detect field type in order to choose usage of keyword for
sort and aggregates
Posted by sh...@apache.org.
UNOMI-70 : use folding + keyword mapping. detect field type in order to choose usage of keyword for sort and aggregates
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/8b5a44bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/8b5a44bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/8b5a44bb
Branch: refs/heads/master
Commit: 8b5a44bb1d9bc0090a6e4d11587d02b542acc567
Parents: 603346a
Author: Thomas Draier <dr...@apache.org>
Authored: Tue Dec 20 11:04:32 2016 +0100
Committer: Thomas Draier <dr...@apache.org>
Committed: Tue Dec 20 11:04:32 2016 +0100
----------------------------------------------------------------------
.../META-INF/cxs/mappings/geonameEntry.json | 48 +--------------
.../META-INF/cxs/mappings/userList.json | 15 -----
.../ElasticSearchPersistenceServiceImpl.java | 64 ++++++++++++++------
.../META-INF/cxs/mappings/_default_.json | 22 +++++++
.../META-INF/cxs/mappings/campaign.json | 24 --------
.../META-INF/cxs/mappings/campaignevent.json | 24 --------
.../resources/META-INF/cxs/mappings/event.json | 18 ------
.../resources/META-INF/cxs/mappings/goal.json | 19 +-----
.../META-INF/cxs/mappings/personaSession.json | 20 +-----
.../META-INF/cxs/mappings/profile.json | 14 +----
.../META-INF/cxs/mappings/propertyType.json | 45 --------------
.../resources/META-INF/cxs/mappings/rule.json | 12 ----
.../META-INF/cxs/mappings/scoring.json | 16 +----
.../META-INF/cxs/mappings/segment.json | 17 +-----
.../META-INF/cxs/mappings/session.json | 25 +-------
.../persistence/spi/PersistenceService.java | 10 +++
.../PropertyConditionESQueryBuilder.java | 4 +-
.../conditions/PropertyConditionEvaluator.java | 12 +++-
.../services/services/GoalsServiceImpl.java | 20 ++++--
.../services/services/QueryServiceImpl.java | 9 ++-
20 files changed, 124 insertions(+), 314 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json
----------------------------------------------------------------------
diff --git a/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json b/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json
index 612f3ca..64038ab 100644
--- a/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json
+++ b/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json
@@ -1,48 +1,10 @@
{
"geonameEntry": {
"properties": {
- "admin1Code": {
- "type": "keyword"
- },
- "admin2Code": {
- "type": "keyword"
- },
- "admin3Code": {
- "type": "keyword"
- },
- "admin4Code": {
- "type": "keyword"
- },
- "alternatenames": {
- "type": "keyword"
- },
- "asciiname": {
- "type": "keyword"
- },
- "cc2": {
- "type": "keyword"
- },
- "countryCode": {
- "type": "keyword"
- },
- "dem": {
- "type": "keyword"
- },
"elevation": {
"type": "long"
},
- "featureClass": {
- "type": "keyword"
- },
- "featureCode": {
- "type": "keyword"
- },
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
+
"location": {
"type": "geo_point"
},
@@ -50,15 +12,11 @@
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
- "name": {
- "type": "keyword"
- },
+
"population": {
"type": "long"
- },
- "timezone": {
- "type": "keyword"
}
+
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json
----------------------------------------------------------------------
diff --git a/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json b/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json
index 53fd043..d73e8b3 100644
--- a/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json
+++ b/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json
@@ -1,12 +1,6 @@
{
"userList": {
"properties": {
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
"metadata": {
"properties": {
"enabled": {
@@ -15,20 +9,11 @@
"hidden": {
"type": "boolean"
},
- "id": {
- "type": "keyword"
- },
"missingPlugins": {
"type": "boolean"
},
"readOnly": {
"type": "boolean"
- },
- "scope": {
- "type": "keyword"
- },
- "tags": {
- "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 5e806d1..ee85d98 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -33,10 +33,7 @@ import org.apache.unomi.api.query.DateRange;
import org.apache.unomi.api.query.IpRange;
import org.apache.unomi.api.query.NumericRange;
import org.apache.unomi.api.services.ClusterService;
-import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder;
-import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher;
-import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator;
-import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorDispatcher;
+import org.apache.unomi.persistence.elasticsearch.conditions.*;
import org.apache.unomi.persistence.spi.CustomObjectMapper;
import org.apache.unomi.persistence.spi.PersistenceService;
import org.apache.unomi.persistence.spi.aggregate.*;
@@ -175,6 +172,8 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private String bulkProcessorFlushInterval = "5s";
private String bulkProcessorBackoffPolicy = "exponential";
+ private Map<String, Map<String, Map<String, Object>>> knownMappings = new HashMap<>();
+
public void setBundleContext(BundleContext bundleContext) {
this.bundleContext = bundleContext;
}
@@ -388,6 +387,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
if (!indexExists) {
logger.info("{} index doesn't exist yet, creating it...", indexName);
Map<String,String> indexMappings = new HashMap<String,String>();
+ indexMappings.put("_default_",mappings.get("_default_"));
for (Map.Entry<String, String> entry : mappings.entrySet()) {
if (!itemsMonthlyIndexed.contains(entry.getKey()) && !indexNames.containsKey(entry.getKey())) {
indexMappings.put(entry.getKey(), entry.getValue());
@@ -631,6 +631,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
logger.info("{} index doesn't exist yet, creating it...", monthlyIndexName);
Map<String,String> indexMappings = new HashMap<String,String>();
+ indexMappings.put("_default_",mappings.get("_default_"));
for (Map.Entry<String, String> entry : mappings.entrySet()) {
if (itemsMonthlyIndexed.contains(entry.getKey())) {
indexMappings.put(entry.getKey(), entry.getValue());
@@ -928,6 +929,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
boolean indexExists = indicesExistsResponse.isExists();
if (!indexExists) {
Map<String,String> indexMappings = new HashMap<String,String>();
+ indexMappings.put("_default_",mappings.get("_default_"));
for (Map.Entry<String, String> entry : mappings.entrySet()) {
if (indexNames.containsKey(entry.getKey()) && indexNames.get(entry.getKey()).equals(indexName)) {
indexMappings.put(entry.getKey(), entry.getValue());
@@ -958,17 +960,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
CreateIndexRequestBuilder builder = client.admin().indices().prepareCreate(indexName)
.setSettings("{\n" +
" \"analysis\": {\n" +
- " \"tokenizer\": {\n" +
- " \"myTokenizer\": {\n" +
- " \"type\":\"pattern\",\n" +
- " \"pattern\":\".*\",\n" +
- " \"group\":0\n" +
- " }\n" +
- " },\n" +
" \"analyzer\": {\n" +
" \"folding\": {\n" +
" \"type\":\"custom\",\n" +
- " \"tokenizer\": \"myTokenizer\",\n" +
+ " \"tokenizer\": \"keyword\",\n" +
" \"filter\": [ \"lowercase\", \"asciifolding\" ]\n" +
" }\n" +
" }\n" +
@@ -995,6 +990,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
@Override
public void createMapping(String type, String source) {
+ if (type.equals("_default_")) {
+ return;
+ }
if (itemsMonthlyIndexed.contains(type)) {
createMapping(type, source, indexName + "-*");
} else if (indexNames.containsKey(type)) {
@@ -1042,6 +1040,36 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}.executeInClassLoader();
}
+ public Map<String, Object> getPropertyMapping(String property, String itemType) {
+ Map<String, Map<String, Object>> mappings = knownMappings.get(itemType);
+ Map<String, Object> result = getPropertyMapping(property, mappings);
+ if (result == null) {
+ mappings = getPropertiesMapping(itemType);
+ knownMappings.put(itemType, mappings);
+ result = getPropertyMapping(property, mappings);
+ }
+ return result;
+ }
+
+ private Map<String, Object> getPropertyMapping(String property, Map<String, Map<String, Object>> mappings) {
+ Map<String, Object> propMapping = null;
+ String[] properties = StringUtils.split(property, '.');
+ for (int i = 0; i < properties.length; i++) {
+ String s = properties[i];
+ if (mappings != null) {
+ propMapping = mappings.get(s);
+ if (i == properties.length - 1) {
+ return propMapping;
+ } else {
+ mappings = (Map<String, Map<String, Object>>) propMapping.get("properties");
+ }
+ } else {
+ return null;
+ }
+ }
+ return propMapping;
+ }
+
public boolean saveQuery(final String queryName, final String query) {
return new InClassLoaderExecute<Boolean>() {
protected Boolean execute(Object... args) {
@@ -1138,12 +1166,12 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
@Override
public <T extends Item> List<T> query(final String fieldName, final String[] fieldValues, String sortBy, final Class<T> clazz) {
- return query(QueryBuilders.termsQuery(fieldName, fieldValues), sortBy, clazz, 0, -1, getRouting(fieldName, fieldValues, clazz), null).getList();
+ return query(QueryBuilders.termsQuery(fieldName, ConditionContextHelper.foldToASCII(fieldValues)), sortBy, clazz, 0, -1, getRouting(fieldName, fieldValues, clazz), null).getList();
}
@Override
public <T extends Item> PartialList<T> query(String fieldName, String fieldValue, String sortBy, Class<T> clazz, int offset, int size) {
- return query(QueryBuilders.termQuery(fieldName, fieldValue), sortBy, clazz, offset, size, getRouting(fieldName, new String[]{fieldValue}, clazz), null);
+ return query(QueryBuilders.termQuery(fieldName, ConditionContextHelper.foldToASCII(fieldValue)), sortBy, clazz, offset, size, getRouting(fieldName, new String[]{fieldValue}, clazz), null);
}
@Override
@@ -1247,9 +1275,11 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
} else {
String name = StringUtils.substringBeforeLast(sortByElement,":");
- if (name.equals("metadata.name") || name.equals("metadata.description")
- || name.equals("properties.firstName") || name.equals("properties.lastName")
- ) {
+ Map<String,Object> propertyMapping = getPropertyMapping(name,itemType);
+ if (propertyMapping != null
+ && "text".equals(propertyMapping.get("type"))
+ && propertyMapping.containsKey("fields")
+ && ((Map)propertyMapping.get("fields")).containsKey("keyword")) {
name += ".keyword";
}
if (sortByElement.endsWith(":desc")) {
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json
new file mode 100644
index 0000000..9bc4ab0
--- /dev/null
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json
@@ -0,0 +1,22 @@
+{
+ "_default_": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
index acd4b85..3e36a7f 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
@@ -4,27 +4,12 @@
"cost": {
"type": "double"
},
- "currency": {
- "type": "keyword"
- },
"startDate": {
"type": "date"
},
"endDate": {
"type": "date"
},
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
- "primaryGoal": {
- "type": "keyword"
- },
- "timezone": {
- "type": "keyword"
- },
"metadata": {
"properties": {
"enabled": {
@@ -33,20 +18,11 @@
"hidden": {
"type": "boolean"
},
- "id": {
- "type": "keyword"
- },
"missingPlugins": {
"type": "boolean"
},
"readOnly": {
"type": "boolean"
- },
- "scope": {
- "type": "keyword"
- },
- "tags": {
- "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
index 7308995..9ab8779 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
@@ -1,27 +1,12 @@
{
"campaignevent": {
"properties": {
- "campaignId": {
- "type": "keyword"
- },
"cost": {
"type": "double"
},
- "currency": {
- "type": "keyword"
- },
"eventDate": {
"type": "date"
},
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
- "timezone": {
- "type": "keyword"
- },
"metadata": {
"properties": {
"enabled": {
@@ -30,20 +15,11 @@
"hidden": {
"type": "boolean"
},
- "id": {
- "type": "keyword"
- },
"missingPlugins": {
"type": "boolean"
},
"readOnly": {
"type": "boolean"
- },
- "scope": {
- "type": "keyword"
- },
- "tags": {
- "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
index c240b3b..69c9f08 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
@@ -1,24 +1,6 @@
{
"event": {
"properties": {
- "eventType": {
- "type": "keyword"
- },
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
- "profileId": {
- "type": "keyword"
- },
- "scope": {
- "type": "keyword"
- },
- "sessionId": {
- "type": "keyword"
- },
"timeStamp": {
"type": "date"
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
index 081915b..3e0bb6f 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
@@ -1,15 +1,7 @@
{
"goal": {
"properties": {
- "campaignId": {
- "type": "keyword"
- },
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
+
"metadata": {
"properties": {
"enabled": {
@@ -18,20 +10,11 @@
"hidden": {
"type": "boolean"
},
- "id": {
- "type": "keyword"
- },
"missingPlugins": {
"type": "boolean"
},
"readOnly": {
"type": "boolean"
- },
- "scope": {
- "type": "keyword"
- },
- "tags": {
- "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
index cb26f1e..ce3a8f3 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
@@ -4,31 +4,13 @@
"duration" : {
"type" : "long"
},
- "itemId" : {
- "type" : "keyword"
- },
- "itemType" : {
- "type" : "keyword"
- },
+
"timeStamp": {
"type": "date"
},
"lastEventDate": {
"type": "date"
},
- "profile" : {
- "properties" : {
- "itemId" : {
- "type" : "keyword"
- },
- "itemType" : {
- "type" : "keyword"
- }
- }
- },
- "profileId" : {
- "type" : "keyword"
- },
"properties" : {
"properties" : {
"location" : {
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
index 665941d..29d40b7 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
@@ -1,12 +1,7 @@
{
"profile": {
"properties": {
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
+
"properties": {
"properties": {
"firstVisit": {
@@ -22,13 +17,6 @@
"type": "long"
}
}
- },
- "systemProperties" : {
- "properties" : {
- "mergeIdentifier" : {
- "type" : "keyword"
- }
- }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
index bdba719..99969f7 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
@@ -1,34 +1,10 @@
{
"propertyType": {
"properties": {
- "automaticMappingsFrom": {
- "type": "keyword"
- },
"dateRanges": {
"properties": {
- "from": {
- "type": "keyword"
- },
- "key": {
- "type": "keyword"
- },
- "to": {
- "type": "keyword"
- }
}
},
- "defaultValue": {
- "type": "keyword"
- },
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
- "mergeStrategy": {
- "type": "keyword"
- },
"metadata": {
"properties": {
"enabled": {
@@ -37,20 +13,11 @@
"hidden": {
"type": "boolean"
},
- "id": {
- "type": "keyword"
- },
"missingPlugins": {
"type": "boolean"
},
"readOnly": {
"type": "boolean"
- },
- "scope": {
- "type": "keyword"
- },
- "tags": {
- "type": "keyword"
}
}
},
@@ -62,9 +29,6 @@
"from": {
"type": "double"
},
- "key": {
- "type": "keyword"
- },
"to": {
"type": "double"
}
@@ -75,15 +39,6 @@
},
"rank": {
"type": "double"
- },
- "tags": {
- "type": "keyword"
- },
- "target": {
- "type": "keyword"
- },
- "type": {
- "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
index aad6029..ea6bb26 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
@@ -1,12 +1,6 @@
{
"rule": {
"properties": {
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
"metadata": {
"properties": {
"enabled": {
@@ -15,17 +9,11 @@
"hidden": {
"type": "boolean"
},
- "id": {
- "type": "keyword"
- },
"missingPlugins": {
"type": "boolean"
},
"readOnly": {
"type": "boolean"
- },
- "scope": {
- "type": "keyword"
}
}
},
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
index b46d7e2..cda229b 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
@@ -1,12 +1,7 @@
{
"scoring": {
"properties": {
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
+
"metadata": {
"properties": {
"enabled": {
@@ -15,20 +10,11 @@
"hidden": {
"type": "boolean"
},
- "id": {
- "type": "keyword"
- },
"missingPlugins": {
"type": "boolean"
},
"readOnly": {
"type": "boolean"
- },
- "scope": {
- "type": "keyword"
- },
- "tags": {
- "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
index 322ff19..5e20436 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
@@ -1,12 +1,7 @@
{
"segment": {
"properties": {
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
+
"metadata": {
"properties": {
"enabled": {
@@ -15,20 +10,12 @@
"hidden": {
"type": "boolean"
},
- "id": {
- "type": "keyword"
- },
+
"missingPlugins": {
"type": "boolean"
},
"readOnly": {
"type": "boolean"
- },
- "scope": {
- "type": "keyword"
- },
- "tags": {
- "type": "keyword"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
index b1759a3..d687dce 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
@@ -4,28 +4,11 @@
"duration": {
"type": "long"
},
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- },
+
"lastEventDate": {
"type": "date"
},
- "profileId": {
- "type": "keyword"
- },
- "profile": {
- "properties": {
- "itemId": {
- "type": "keyword"
- },
- "itemType": {
- "type": "keyword"
- }
- }
- },
+
"properties": {
"properties": {
"location": {
@@ -33,9 +16,7 @@
}
}
},
- "scope": {
- "type": "keyword"
- },
+
"size": {
"type": "long"
},
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
----------------------------------------------------------------------
diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
index e191829..132c583 100644
--- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
+++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
@@ -174,6 +174,16 @@ public interface PersistenceService {
Map<String, Map<String, Object>> getPropertiesMapping(String itemType);
/**
+ * Retrieve the mapping for one specific property for a given type.
+ *
+ * @param property the property name (can use nested dot notation)
+ * @param itemType the itemType we want to retrieve the mappings for
+ * @return
+ */
+ Map<String, Object> getPropertyMapping(String property, String itemType);
+
+
+ /**
* Create mapping
* @param type
* @param source
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
index 31aa0a3..c36722b 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
@@ -44,12 +44,12 @@ public class PropertyConditionESQueryBuilder implements ConditionESQueryBuilder
throw new IllegalArgumentException("Impossible to build ES filter, condition is not valid, comparisonOperator and propertyName properties should be provided");
}
- String expectedValue = (String) condition.getParameter("propertyValue");
+ String expectedValue = ConditionContextHelper.foldToASCII((String) condition.getParameter("propertyValue"));
Object expectedValueInteger = condition.getParameter("propertyValueInteger");
Object expectedValueDate = condition.getParameter("propertyValueDate");
Object expectedValueDateExpr = condition.getParameter("propertyValueDateExpr");
- List<?> expectedValues = (List<?>) condition.getParameter("propertyValues");
+ List<?> expectedValues = ConditionContextHelper.foldToASCII((List<?>) condition.getParameter("propertyValues"));
List<?> expectedValuesInteger = (List<?>) condition.getParameter("propertyValuesInteger");
List<?> expectedValuesDate = (List<?>) condition.getParameter("propertyValuesDate");
List<?> expectedValuesDateExpr = (List<?>) condition.getParameter("propertyValuesDateExpr");
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
index d8b7754..68e58a6 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
@@ -83,7 +83,7 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
return false;
}
- List<Object> actual = getValueSet(actualValue);
+ List<Object> actual = ConditionContextHelper.foldToASCII(getValueSet(actualValue));
boolean result = true;
@@ -136,7 +136,7 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
String op = (String) condition.getParameter("comparisonOperator");
String name = (String) condition.getParameter("propertyName");
- String expectedValue = (String) condition.getParameter("propertyValue");
+ String expectedValue = ConditionContextHelper.foldToASCII((String) condition.getParameter("propertyValue"));
Object expectedValueInteger = condition.getParameter("propertyValueInteger");
Object expectedValueDate = condition.getParameter("propertyValueDate");
Object expectedValueDateExpr = condition.getParameter("propertyValueDateExpr");
@@ -165,6 +165,9 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
actualValue = null;
}
}
+ if (actualValue instanceof String) {
+ actualValue = ConditionContextHelper.foldToASCII((String) actualValue);
+ }
if(op == null) {
return false;
@@ -175,6 +178,9 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
} else if (op.equals("equals")) {
if (actualValue instanceof Collection) {
for (Object o : ((Collection<?>)actualValue)) {
+ if (o instanceof String) {
+ o = ConditionContextHelper.foldToASCII((String) o);
+ }
if (compare(o, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr) == 0) {
return true;
}
@@ -214,7 +220,7 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
} else if (op.equals("matchesRegex")) {
return expectedValue != null && Pattern.compile(expectedValue).matcher(actualValue.toString()).matches();
} else if (op.equals("in") || op.equals("notIn") || op.equals("hasSomeOf") || op.equals("hasNoneOf") || op.equals("all")) {
- List<?> expectedValues = (List<?>) condition.getParameter("propertyValues");
+ List<?> expectedValues = ConditionContextHelper.foldToASCII((List<?>) condition.getParameter("propertyValues"));
List<?> expectedValuesInteger = (List<?>) condition.getParameter("propertyValuesInteger");
List<?> expectedValuesDate = (List<?>) condition.getParameter("propertyValuesDate");
List<?> expectedValuesDateExpr = (List<?>) condition.getParameter("propertyValuesDateExpr");
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java
----------------------------------------------------------------------
diff --git a/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java
index a99f155..a6bd1d6 100644
--- a/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java
@@ -474,25 +474,33 @@ public class GoalsServiceImpl implements GoalsService, SynchronousBundleListener
// resolve aggregate
BaseAggregate aggregate = null;
- if(query != null && query.getAggregate() != null && query.getAggregate().getProperty() != null) {
+ String property = query.getAggregate().getProperty();
+ if(query != null && query.getAggregate() != null && property != null) {
if (query.getAggregate().getType() != null){
// try to guess the aggregate type
if(query.getAggregate().getType().equals("date")) {
String interval = (String) query.getAggregate().getParameters().get("interval");
String format = (String) query.getAggregate().getParameters().get("format");
- aggregate = new DateAggregate(query.getAggregate().getProperty(), interval, format);
+ aggregate = new DateAggregate(property, interval, format);
} else if (query.getAggregate().getType().equals("dateRange") && query.getAggregate().getDateRanges() != null && query.getAggregate().getDateRanges().size() > 0) {
String format = (String) query.getAggregate().getParameters().get("format");
- aggregate = new DateRangeAggregate(query.getAggregate().getProperty(), format, query.getAggregate().getDateRanges());
+ aggregate = new DateRangeAggregate(property, format, query.getAggregate().getDateRanges());
} else if (query.getAggregate().getType().equals("numericRange") && query.getAggregate().getNumericRanges() != null && query.getAggregate().getNumericRanges().size() > 0) {
- aggregate = new NumericRangeAggregate(query.getAggregate().getProperty(), query.getAggregate().getNumericRanges());
+ aggregate = new NumericRangeAggregate(property, query.getAggregate().getNumericRanges());
} else if (query.getAggregate().getType().equals("ipRange") && query.getAggregate().ipRanges() != null && query.getAggregate().ipRanges().size() > 0) {
- aggregate = new IpRangeAggregate(query.getAggregate().getProperty(), query.getAggregate().ipRanges());
+ aggregate = new IpRangeAggregate(property, query.getAggregate().ipRanges());
}
}
if(aggregate == null){
- aggregate = new TermsAggregate(query.getAggregate().getProperty() + ".keyword");
+ Map<String,Object> propertyMapping = persistenceService.getPropertyMapping(property, Session.ITEM_TYPE);
+ if (propertyMapping != null
+ && "text".equals(propertyMapping.get("type"))
+ && propertyMapping.containsKey("fields")
+ && ((Map)propertyMapping.get("fields")).containsKey("keyword")) {
+ property += ".keyword";
+ }
+ aggregate = new TermsAggregate(property);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
----------------------------------------------------------------------
diff --git a/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
index 79edd43..30e6dab 100644
--- a/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
@@ -83,7 +83,14 @@ public class QueryServiceImpl implements QueryService {
}
// fall back on terms aggregate
- return persistenceService.aggregateQuery(query.getCondition(), new TermsAggregate(property + ".keyword"), itemType);
+ Map<String,Object> propertyMapping = persistenceService.getPropertyMapping( property, itemType);
+ if (propertyMapping != null
+ && "text".equals(propertyMapping.get("type"))
+ && propertyMapping.containsKey("fields")
+ && ((Map)propertyMapping.get("fields")).containsKey("keyword")) {
+ property += ".keyword";
+ }
+ return persistenceService.aggregateQuery(query.getCondition(), new TermsAggregate(property), itemType);
}
return getAggregate(itemType, property);
[03/50] [abbrv] incubator-unomi git commit:
[jgitflow-maven-plugin]updating poms for unomi_28_es_2_x version
Posted by sh...@apache.org.
[jgitflow-maven-plugin]updating poms for unomi_28_es_2_x version
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/9493c703
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/9493c703
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/9493c703
Branch: refs/heads/master
Commit: 9493c70307cb845fe7628eb4a924cac0a3df1de2
Parents: b46a599
Author: Serge Huber <sh...@apache.org>
Authored: Wed Nov 16 14:31:34 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Wed Nov 16 14:31:34 2016 +0100
----------------------------------------------------------------------
api/pom.xml | 2 +-
extensions/geonames/pom.xml | 2 +-
extensions/geonames/rest/pom.xml | 8 +++---
extensions/geonames/services/pom.xml | 8 +++---
extensions/lists-extension/pom.xml | 4 +--
extensions/lists-extension/rest/pom.xml | 8 +++---
extensions/lists-extension/services/pom.xml | 8 +++---
extensions/pom.xml | 2 +-
extensions/privacy-extension/pom.xml | 4 +--
extensions/privacy-extension/rest/pom.xml | 8 +++---
extensions/privacy-extension/services/pom.xml | 8 +++---
itests/pom.xml | 8 +++---
kar/pom.xml | 30 ++++++++++----------
package/pom.xml | 4 +--
performance-tests/pom.xml | 4 +--
persistence-elasticsearch/core/pom.xml | 6 ++--
persistence-elasticsearch/plugins/pom.xml | 2 +-
.../plugins/security/pom.xml | 2 +-
persistence-elasticsearch/pom.xml | 2 +-
persistence-spi/pom.xml | 4 +--
plugins/baseplugin/pom.xml | 6 ++--
plugins/hover-event/pom.xml | 4 +--
plugins/mail/pom.xml | 4 +--
plugins/optimization-test/pom.xml | 2 +-
plugins/past-event/pom.xml | 6 ++--
plugins/pom.xml | 4 +--
plugins/request/pom.xml | 2 +-
plugins/tracked-event/pom.xml | 6 ++--
pom.xml | 2 +-
rest/pom.xml | 6 ++--
samples/pom.xml | 2 +-
samples/tweet-button-plugin/pom.xml | 4 +--
services/pom.xml | 6 ++--
wab/pom.xml | 6 ++--
34 files changed, 92 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/api/pom.xml
----------------------------------------------------------------------
diff --git a/api/pom.xml b/api/pom.xml
index 0f06620..c49ce5a 100644
--- a/api/pom.xml
+++ b/api/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/extensions/geonames/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/geonames/pom.xml b/extensions/geonames/pom.xml
index f601a7c..abe9f22 100644
--- a/extensions/geonames/pom.xml
+++ b/extensions/geonames/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-extensions</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>cxs-geonames</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/extensions/geonames/rest/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/geonames/rest/pom.xml b/extensions/geonames/rest/pom.xml
index 6798bfe..e4dbdd6 100644
--- a/extensions/geonames/rest/pom.xml
+++ b/extensions/geonames/rest/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-geonames</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -33,14 +33,14 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-geonames-services</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
@@ -81,7 +81,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/extensions/geonames/services/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/geonames/services/pom.xml b/extensions/geonames/services/pom.xml
index e1e7310..8b5e67e 100644
--- a/extensions/geonames/services/pom.xml
+++ b/extensions/geonames/services/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-geonames</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -28,20 +28,20 @@
<artifactId>cxs-geonames-services</artifactId>
<name>Apache Unomi :: Extensions :: Geonames Database :: Service</name>
<description>Service implementation for the Apache Unomi Context Server extension that integrates with the Geonames database</description>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<packaging>bundle</packaging>
<dependencies>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/extensions/lists-extension/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/lists-extension/pom.xml b/extensions/lists-extension/pom.xml
index d348ad9..f1441b7 100644
--- a/extensions/lists-extension/pom.xml
+++ b/extensions/lists-extension/pom.xml
@@ -27,12 +27,12 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-extensions</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>cxs-lists-extension</artifactId>
<name>Apache Unomi :: Extensions :: Lists</name>
<description>List extension for the Apache Unomi Context Server</description>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<packaging>pom</packaging>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/extensions/lists-extension/rest/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/lists-extension/rest/pom.xml b/extensions/lists-extension/rest/pom.xml
index 7a1f6b6..40fa1d5 100644
--- a/extensions/lists-extension/rest/pom.xml
+++ b/extensions/lists-extension/rest/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>cxs-lists-extension</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -34,14 +34,14 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-lists-extension-services</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
@@ -82,7 +82,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/extensions/lists-extension/services/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/lists-extension/services/pom.xml b/extensions/lists-extension/services/pom.xml
index d7ebedb..353fad3 100644
--- a/extensions/lists-extension/services/pom.xml
+++ b/extensions/lists-extension/services/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>cxs-lists-extension</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -28,20 +28,20 @@
<name>Apache Unomi :: Extensions :: Lists :: Service</name>
<description>List extension service implementation for the Apache Unomi Context Server</description>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<packaging>bundle</packaging>
<dependencies>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/extensions/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/pom.xml b/extensions/pom.xml
index aa9cb48..6043137 100644
--- a/extensions/pom.xml
+++ b/extensions/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-extensions</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/extensions/privacy-extension/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/privacy-extension/pom.xml b/extensions/privacy-extension/pom.xml
index 46e4315..76f18a8 100644
--- a/extensions/privacy-extension/pom.xml
+++ b/extensions/privacy-extension/pom.xml
@@ -27,12 +27,12 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-extensions</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>cxs-privacy-extension</artifactId>
<name>Apache Unomi :: Extensions :: Privacy</name>
<description>Privacy management extension for the Apache Unomi Context Server</description>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<packaging>pom</packaging>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/extensions/privacy-extension/rest/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/privacy-extension/rest/pom.xml b/extensions/privacy-extension/rest/pom.xml
index d6800cf..f9dac53 100644
--- a/extensions/privacy-extension/rest/pom.xml
+++ b/extensions/privacy-extension/rest/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>cxs-privacy-extension</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -33,14 +33,14 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-privacy-extension-services</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
@@ -81,7 +81,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/extensions/privacy-extension/services/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/privacy-extension/services/pom.xml b/extensions/privacy-extension/services/pom.xml
index 4afcde8..1ee0b5d 100644
--- a/extensions/privacy-extension/services/pom.xml
+++ b/extensions/privacy-extension/services/pom.xml
@@ -20,27 +20,27 @@
<parent>
<artifactId>cxs-privacy-extension</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>cxs-privacy-extension-services</artifactId>
<name>Apache Unomi :: Extensions :: Privacy :: Services</name>
<description>Privacy management extension service implementation for the Apache Unomi Context Server</description>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<packaging>bundle</packaging>
<dependencies>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/itests/pom.xml
----------------------------------------------------------------------
diff --git a/itests/pom.xml b/itests/pom.xml
index e0426ce..7b89bb9 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-itests</artifactId>
<name>Apache Unomi :: Integration Tests</name>
@@ -31,19 +31,19 @@
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-kar</artifactId>
<classifier>features</classifier>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<type>xml</type>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-wab</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/kar/pom.xml
----------------------------------------------------------------------
diff --git a/kar/pom.xml b/kar/pom.xml
index 7d6c400..6d3206a 100644
--- a/kar/pom.xml
+++ b/kar/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-kar</artifactId>
@@ -33,72 +33,72 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-wab</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-services</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins-base</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins-request</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins-mail</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins-optimization-test</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-lists-extension-services</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-lists-extension-rest</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-geonames-services</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-geonames-rest</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-privacy-extension-services</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-privacy-extension-rest</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.servicemix.bundles</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/package/pom.xml
----------------------------------------------------------------------
diff --git a/package/pom.xml b/package/pom.xml
index b3f7448..c1c1eaa 100644
--- a/package/pom.xml
+++ b/package/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi</artifactId>
@@ -75,7 +75,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-kar</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<classifier>features</classifier>
<type>xml</type>
<scope>runtime</scope>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/performance-tests/pom.xml
----------------------------------------------------------------------
diff --git a/performance-tests/pom.xml b/performance-tests/pom.xml
index 72c7abd..ae9d6ff 100644
--- a/performance-tests/pom.xml
+++ b/performance-tests/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-performance-tests</artifactId>
<name>Apache Unomi :: Performance Tests</name>
@@ -30,7 +30,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-wab</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/persistence-elasticsearch/core/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/pom.xml b/persistence-elasticsearch/core/pom.xml
index 891da68..506a5c7 100644
--- a/persistence-elasticsearch/core/pom.xml
+++ b/persistence-elasticsearch/core/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
@@ -39,13 +39,13 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/persistence-elasticsearch/plugins/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/plugins/pom.xml b/persistence-elasticsearch/plugins/pom.xml
index 9e361c0..3df1d5e 100644
--- a/persistence-elasticsearch/plugins/pom.xml
+++ b/persistence-elasticsearch/plugins/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-persistence-elasticsearch-plugins</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/persistence-elasticsearch/plugins/security/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/plugins/security/pom.xml b/persistence-elasticsearch/plugins/security/pom.xml
index b029a8c..3dffda7 100644
--- a/persistence-elasticsearch/plugins/security/pom.xml
+++ b/persistence-elasticsearch/plugins/security/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-plugins</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-persistence-elasticsearch-plugins-security</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/persistence-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/pom.xml b/persistence-elasticsearch/pom.xml
index 99ef177..05fd149 100644
--- a/persistence-elasticsearch/pom.xml
+++ b/persistence-elasticsearch/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-persistence-elasticsearch</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/persistence-spi/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-spi/pom.xml b/persistence-spi/pom.xml
index 68d5ba7..5a770ca 100644
--- a/persistence-spi/pom.xml
+++ b/persistence-spi/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-persistence-spi</artifactId>
@@ -33,7 +33,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/plugins/baseplugin/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/pom.xml b/plugins/baseplugin/pom.xml
index 078cfb8..9d17d75 100644
--- a/plugins/baseplugin/pom.xml
+++ b/plugins/baseplugin/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-base</artifactId>
@@ -55,13 +55,13 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/plugins/hover-event/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/hover-event/pom.xml b/plugins/hover-event/pom.xml
index 979aa35..1b23db4 100644
--- a/plugins/hover-event/pom.xml
+++ b/plugins/hover-event/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-hover-event</artifactId>
@@ -33,7 +33,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/plugins/mail/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/mail/pom.xml b/plugins/mail/pom.xml
index 2bdcdd3..458cce0 100644
--- a/plugins/mail/pom.xml
+++ b/plugins/mail/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-mail</artifactId>
@@ -33,7 +33,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/plugins/optimization-test/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/optimization-test/pom.xml b/plugins/optimization-test/pom.xml
index f034ed2..e867e88 100644
--- a/plugins/optimization-test/pom.xml
+++ b/plugins/optimization-test/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-optimization-test</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/plugins/past-event/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/past-event/pom.xml b/plugins/past-event/pom.xml
index 7f63535..4966b8c 100644
--- a/plugins/past-event/pom.xml
+++ b/plugins/past-event/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-past-event</artifactId>
@@ -33,13 +33,13 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/plugins/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/pom.xml b/plugins/pom.xml
index 35d1fbd..101a861 100644
--- a/plugins/pom.xml
+++ b/plugins/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins</artifactId>
@@ -43,7 +43,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/plugins/request/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/request/pom.xml b/plugins/request/pom.xml
index 1764ed1..0144501 100644
--- a/plugins/request/pom.xml
+++ b/plugins/request/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-request</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/plugins/tracked-event/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/tracked-event/pom.xml b/plugins/tracked-event/pom.xml
index bb47811..fa3b5d5 100644
--- a/plugins/tracked-event/pom.xml
+++ b/plugins/tracked-event/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-tracked-event</artifactId>
@@ -33,13 +33,13 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e6a8f4a..bb44fb2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -34,7 +34,7 @@
provides a high-performance user profile and event tracking server.
</description>
<url>http://unomi.incubator.apache.org</url>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<packaging>pom</packaging>
<licenses>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/rest/pom.xml
----------------------------------------------------------------------
diff --git a/rest/pom.xml b/rest/pom.xml
index bba4a5a..5c33145 100644
--- a/rest/pom.xml
+++ b/rest/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-rest</artifactId>
@@ -46,7 +46,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
@@ -93,7 +93,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<!--<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/samples/pom.xml
----------------------------------------------------------------------
diff --git a/samples/pom.xml b/samples/pom.xml
index 8941427..419ff06 100644
--- a/samples/pom.xml
+++ b/samples/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>unomi-root</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/samples/tweet-button-plugin/pom.xml
----------------------------------------------------------------------
diff --git a/samples/tweet-button-plugin/pom.xml b/samples/tweet-button-plugin/pom.xml
index c1725d5..4632d38 100644
--- a/samples/tweet-button-plugin/pom.xml
+++ b/samples/tweet-button-plugin/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>samples</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>tweet-button-plugin</artifactId>
@@ -33,7 +33,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/services/pom.xml
----------------------------------------------------------------------
diff --git a/services/pom.xml b/services/pom.xml
index abd3cc1..55a2f62 100644
--- a/services/pom.xml
+++ b/services/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-services</artifactId>
@@ -33,7 +33,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
@@ -52,7 +52,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/9493c703/wab/pom.xml
----------------------------------------------------------------------
diff --git a/wab/pom.xml b/wab/pom.xml
index 5af699f..d3bc9e1 100644
--- a/wab/pom.xml
+++ b/wab/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-wab</artifactId>
@@ -42,13 +42,13 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_28_es_2_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
[34/50] [abbrv] incubator-unomi git commit: Update feature to add
dependency to cellar
Posted by sh...@apache.org.
Update feature to add dependency to cellar
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/207c2cc4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/207c2cc4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/207c2cc4
Branch: refs/heads/master
Commit: 207c2cc429ce989e6a8939e1ceccf89fcf02dfce
Parents: 31677cb
Author: Serge Huber <sh...@apache.org>
Authored: Wed Dec 21 20:15:50 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Wed Dec 21 20:15:50 2016 +0100
----------------------------------------------------------------------
kar/src/main/feature/feature.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/207c2cc4/kar/src/main/feature/feature.xml
----------------------------------------------------------------------
diff --git a/kar/src/main/feature/feature.xml b/kar/src/main/feature/feature.xml
index 39413fb..674e6bc 100644
--- a/kar/src/main/feature/feature.xml
+++ b/kar/src/main/feature/feature.xml
@@ -23,6 +23,7 @@
<feature>cxf</feature>
<feature>openwebbeans</feature>
<feature>pax-cdi-web-openwebbeans</feature>
+ <feature>cellar</feature>
<configfile finalname="/etc/org.apache.unomi.web.cfg">mvn:org.apache.unomi/unomi-wab/${project.version}/cfg/unomicfg</configfile>
<configfile finalname="/etc/org.apache.unomi.persistence.elasticsearch.cfg">mvn:org.apache.unomi/unomi-persistence-elasticsearch-core/${project.version}/cfg/elasticsearchcfg</configfile>
<configfile finalname="/etc/org.apache.unomi.plugins.request.cfg">mvn:org.apache.unomi/unomi-plugins-request/${project.version}/cfg/requestcfg</configfile>
[22/50] [abbrv] incubator-unomi git commit: Merge branch
'feature-UNOMI-28-ES2X' into feature-UNOMI-70-ES5X
Posted by sh...@apache.org.
Merge branch 'feature-UNOMI-28-ES2X' into feature-UNOMI-70-ES5X
# Conflicts:
# persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/923e4917
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/923e4917
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/923e4917
Branch: refs/heads/master
Commit: 923e49171fae139e2197b55ef4b5102d61221edc
Parents: a94fdb0 28d7dbd
Author: Serge Huber <sh...@apache.org>
Authored: Thu Dec 15 16:40:41 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Thu Dec 15 16:40:41 2016 +0100
----------------------------------------------------------------------
.../java/org/apache/unomi/api/PartialList.java | 25 +++++
buildAndRunNoTests.sh | 34 ++++++
.../ElasticSearchPersistenceServiceImpl.java | 95 +++++++++++++---
.../persistence/spi/PersistenceService.java | 33 ++++++
.../PropertyConditionESQueryBuilder.java | 3 +-
.../services/services/SegmentServiceImpl.java | 111 ++++++++++++-------
.../resources/OSGI-INF/blueprint/blueprint.xml | 2 +
7 files changed, 242 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/923e4917/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --cc persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index fa4b5e1,d6a136e..0651c3f
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@@ -699,7 -711,7 +699,7 @@@ public class ElasticSearchPersistenceSe
String itemType = (String) clazz.getField("ITEM_TYPE").get(null);
if (itemsMonthlyIndexed.contains(itemType) && dateHint == null) {
- PartialList<T> r = query(QueryBuilders.idsQuery(itemType).addIds(itemId), null, clazz, 0, 1, null);
- PartialList<T> r = query(QueryBuilders.idsQuery(itemType).ids(itemId), null, clazz, 0, 1, null, null);
++ PartialList<T> r = query(QueryBuilders.idsQuery(itemType).addIds(itemId), null, clazz, 0, 1, null, null);
if (r.size() > 0) {
return r.get(0);
}
@@@ -1238,35 -1262,14 +1256,36 @@@
SearchResponse response = requestBuilder
.execute()
.actionGet();
- SearchHits searchHits = response.getHits();
- scrollIdentifier = response.getScrollId();
- totalHits = searchHits.getTotalHits();
- for (SearchHit searchHit : searchHits) {
- String sourceAsString = searchHit.getSourceAsString();
- final T value = CustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz);
- value.setItemId(searchHit.getId());
- results.add(value);
+ if (size == -1) {
+ // Scroll until no more hits are returned
+ while (true) {
+
+ for (SearchHit searchHit : response.getHits().getHits()) {
+ // add hit to results
+ String sourceAsString = searchHit.getSourceAsString();
+ final T value = CustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz);
+ value.setItemId(searchHit.getId());
+ results.add(value);
+ }
+
+ response = client.prepareSearchScroll(response.getScrollId()).setScroll(keepAlive).execute().actionGet();
+
+ // If we have no more hits, exit
+ if (response.getHits().getHits().length == 0) {
+ break;
+ }
+ }
+ client.prepareClearScroll().addScrollId(response.getScrollId()).execute().actionGet();
+ } else {
+ SearchHits searchHits = response.getHits();
++ scrollIdentifier = response.getScrollId();
+ totalHits = searchHits.getTotalHits();
+ for (SearchHit searchHit : searchHits) {
+ String sourceAsString = searchHit.getSourceAsString();
+ final T value = CustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz);
+ value.setItemId(searchHit.getId());
+ results.add(value);
+ }
}
} catch (Exception t) {
logger.error("Error loading itemType=" + clazz.getName() + " query=" + query + " sortBy=" + sortBy, t);
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/923e4917/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
----------------------------------------------------------------------
[24/50] [abbrv] incubator-unomi git commit: Document new segment
update parameter
Posted by sh...@apache.org.
Document new segment update parameter
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/ca5c9901
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/ca5c9901
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/ca5c9901
Branch: refs/heads/master
Commit: ca5c99011a54c802c1542309d4726889be8445b4
Parents: 1cd827a
Author: Serge Huber <sh...@apache.org>
Authored: Fri Dec 16 11:19:15 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Fri Dec 16 11:19:15 2016 +0100
----------------------------------------------------------------------
services/src/main/resources/org.apache.unomi.services.cfg | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ca5c9901/services/src/main/resources/org.apache.unomi.services.cfg
----------------------------------------------------------------------
diff --git a/services/src/main/resources/org.apache.unomi.services.cfg b/services/src/main/resources/org.apache.unomi.services.cfg
index 380914a..d69da60 100644
--- a/services/src/main/resources/org.apache.unomi.services.cfg
+++ b/services/src/main/resources/org.apache.unomi.services.cfg
@@ -26,3 +26,7 @@ profile.purge.existTime=-1
# Purge all sessions/events that have been created for a specific number of months
event.purge.existTime=12
+
+# When performing segment updates, this controls the size of the scrolling query size used to iterate over all the
+# profiles that need updating
+segment.update.batchSize=1000
[38/50] [abbrv] incubator-unomi git commit: - Fix waiting for green
status message - Add code to register mappings and condition builders that
were started before the ElasticSearch persistence service implementation -
Change counting code to use queries
Posted by sh...@apache.org.
- Fix waiting for green status message
- Add code to register mappings and condition builders that were started before the ElasticSearch persistence service implementation
- Change counting code to use queries instead of aggregations
Signed-off-by: Serge Huber <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/35b93979
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/35b93979
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/35b93979
Branch: refs/heads/master
Commit: 35b9397931cd89dd6d021e65079c658d6fdf29e0
Parents: 952da61
Author: Serge Huber <sh...@apache.org>
Authored: Wed Dec 21 21:29:13 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Wed Dec 21 21:29:13 2016 +0100
----------------------------------------------------------------------
.../ElasticSearchPersistenceServiceImpl.java | 39 ++++++++++++++------
1 file changed, 27 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/35b93979/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 6c1d8e0..dc909b9 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -24,7 +24,10 @@ import org.apache.karaf.cellar.core.*;
import org.apache.karaf.cellar.core.control.SwitchStatus;
import org.apache.karaf.cellar.core.event.EventProducer;
import org.apache.karaf.cellar.core.event.EventType;
-import org.apache.unomi.api.*;
+import org.apache.unomi.api.ClusterNode;
+import org.apache.unomi.api.Item;
+import org.apache.unomi.api.PartialList;
+import org.apache.unomi.api.TimestampedItem;
import org.apache.unomi.api.conditions.Condition;
import org.apache.unomi.api.query.DateRange;
import org.apache.unomi.api.query.IpRange;
@@ -79,10 +82,7 @@ import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
-import org.osgi.framework.BundleContext;
-import org.osgi.framework.BundleEvent;
-import org.osgi.framework.ServiceReference;
-import org.osgi.framework.SynchronousBundleListener;
+import org.osgi.framework.*;
import org.osgi.service.cm.ConfigurationAdmin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -415,7 +415,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
refreshExistingIndexNames();
- logger.info("Waiting for index creation to complete...");
+ logger.info("Waiting for GREEN cluster status...");
client.admin().cluster().prepareHealth()
.setWaitForGreenStatus()
@@ -458,6 +458,24 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
}, 10000L, 24L * 60L * 60L * 1000L);
+ // load predefined mappings and condition dispatchers of any bundles that were started before this one.
+ for (Bundle existingBundle : bundleContext.getBundles()) {
+ if (existingBundle.getBundleContext() != null) {
+ loadPredefinedMappings(existingBundle.getBundleContext(), true);
+ }
+ if (existingBundle.getRegisteredServices() != null) {
+ for (ServiceReference<?> reference : existingBundle.getRegisteredServices()) {
+ Object service = bundleContext.getService(reference);
+ if (service instanceof ConditionEvaluator) {
+ conditionEvaluatorDispatcher.addEvaluator(reference.getProperty("conditionEvaluatorId").toString(), existingBundle.getBundleId(), (ConditionEvaluator) service);
+ }
+ if (service instanceof ConditionESQueryBuilder) {
+ conditionESQueryBuilderDispatcher.addQueryBuilder(reference.getProperty("queryBuilderId").toString(), existingBundle.getBundleId(), (ConditionESQueryBuilder) service);
+ }
+ }
+ }
+ }
+
logger.info(this.getClass().getName() + " service started successfully.");
}
@@ -649,7 +667,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
while (predefinedMappings.hasMoreElements()) {
URL predefinedMappingURL = predefinedMappings.nextElement();
- logger.debug("Found mapping at " + predefinedMappingURL + ", loading... ");
+ logger.info("Found mapping at " + predefinedMappingURL + ", loading... ");
try {
final String path = predefinedMappingURL.getPath();
String name = path.substring(path.lastIndexOf('/') + 1, path.lastIndexOf('.'));
@@ -1218,13 +1236,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
SearchResponse response = client.prepareSearch(getIndexNameForQuery(itemType))
.setTypes(itemType)
.setSize(0)
- .setQuery(QueryBuilders.matchAllQuery())
- .addAggregation(AggregationBuilders.filter("filter", filter))
+ .setQuery(filter)
.execute()
.actionGet();
- Aggregations searchHits = response.getAggregations();
- Filter filter = searchHits.get("filter");
- return filter.getDocCount();
+ return response.getHits().getTotalHits();
}
}.executeInClassLoader();
}
[11/50] [abbrv] incubator-unomi git commit: Merge branch 'master'
into feature-UNOMI-28-ES2X
Posted by sh...@apache.org.
Merge branch 'master' into feature-UNOMI-28-ES2X
# Conflicts:
# plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/2524693c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/2524693c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/2524693c
Branch: refs/heads/master
Commit: 2524693c34beffddece4ee1b520e48aafe9552e0
Parents: 85f00e8 173c1eb
Author: Serge Huber <sh...@apache.org>
Authored: Fri Nov 25 14:16:13 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Fri Nov 25 14:16:13 2016 +0100
----------------------------------------------------------------------
.../unomi/privacy/internal/PrivacyServiceImpl.java | 2 +-
.../apache/unomi/itests/ConditionEvaluatorTest.java | 2 +-
.../conditions/PropertyConditionESQueryBuilder.java | 14 ++++++++++++++
.../conditions/PropertyConditionEvaluator.java | 12 +++++++++++-
4 files changed, 27 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/2524693c/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --cc plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
index 23f1df8,d2f77c0..cf58734
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
@@@ -61,54 -60,56 +61,68 @@@ public class PropertyConditionESQueryBu
switch (op) {
case "equals":
checkRequiredValue(value, name, op, false);
- return FilterBuilders.termFilter(name, value);
+ return QueryBuilders.termQuery(name, value);
case "notEquals":
checkRequiredValue(value, name, op, false);
- return FilterBuilders.notFilter(FilterBuilders.termFilter(name, value));
+ return QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(name, value));
case "greaterThan":
checkRequiredValue(value, name, op, false);
- return FilterBuilders.rangeFilter(name).gt(value);
+ return QueryBuilders.rangeQuery(name).gt(value);
case "greaterThanOrEqualTo":
checkRequiredValue(value, name, op, false);
- return FilterBuilders.rangeFilter(name).gte(value);
+ return QueryBuilders.rangeQuery(name).gte(value);
case "lessThan":
checkRequiredValue(value, name, op, false);
- return FilterBuilders.rangeFilter(name).lt(value);
+ return QueryBuilders.rangeQuery(name).lt(value);
case "lessThanOrEqualTo":
checkRequiredValue(value, name, op, false);
- return FilterBuilders.rangeFilter(name).lte(value);
+ return QueryBuilders.rangeQuery(name).lte(value);
case "between":
checkRequiredValuesSize(values, name, op, 2);
- return FilterBuilders.rangeFilter(name).gte(values.get(0)).lte(values.get(1));
+ return QueryBuilders.rangeQuery(name).gte(values.get(0)).lte(values.get(1));
case "exists":
- return FilterBuilders.existsFilter(name);
+ return QueryBuilders.existsQuery(name);
case "missing":
- return FilterBuilders.missingFilter(name);
+ return QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery((name)));
case "contains":
checkRequiredValue(expectedValue, name, op, false);
- return FilterBuilders.regexpFilter(name, ".*" + expectedValue + ".*");
+ return QueryBuilders.regexpQuery(name, ".*" + expectedValue + ".*");
case "startsWith":
checkRequiredValue(expectedValue, name, op, false);
- return FilterBuilders.prefixFilter(name, expectedValue);
+ return QueryBuilders.prefixQuery(name, expectedValue);
case "endsWith":
checkRequiredValue(expectedValue, name, op, false);
- return FilterBuilders.regexpFilter(name, ".*" + expectedValue);
+ return QueryBuilders.regexpQuery(name, ".*" + expectedValue);
case "matchesRegex":
checkRequiredValue(expectedValue, name, op, false);
- return FilterBuilders.regexpFilter(name, expectedValue);
+ return QueryBuilders.regexpQuery(name, expectedValue);
case "in":
checkRequiredValue(values, name, op, true);
- return FilterBuilders.inFilter(name, values.toArray());
+ return QueryBuilders.termsQuery(name, values.toArray());
case "notIn":
checkRequiredValue(values, name, op, true);
- return FilterBuilders.notFilter(FilterBuilders.inFilter(name, values.toArray()));
+ return QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(name, values.toArray()));
case "all":
checkRequiredValue(values, name, op, true);
- return FilterBuilders.termsFilter(name, values.toArray()).execution("and");
+ BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+ for (Object curValue : values) {
+ boolQueryBuilder.must(QueryBuilders.termQuery(name, curValue));
+ }
+ return boolQueryBuilder;
+ case "hasSomeOf":
+ checkRequiredValue(values, name, op, true);
- return FilterBuilders.termsFilter(name, values.toArray()).execution("or");
++ BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
++ for (Object curValue : values) {
++ boolQueryBuilder.should(QueryBuilders.termQuery(name, curValue));
++ }
++ return boolQueryBuilder;
+ case "hasNoneOf":
+ checkRequiredValue(values, name, op, true);
- return FilterBuilders.notFilter(FilterBuilders.termsFilter(name, values.toArray()).execution("or"));
++ BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
++ for (Object curValue : values) {
++ boolQueryBuilder.mustNot(QueryBuilders.termQuery(name, curValue));
++ }
++ return boolQueryBuilder;
case "isDay":
checkRequiredValue(value, name, op, false);
return getIsSameDayRange(value, name);
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/2524693c/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
----------------------------------------------------------------------
[09/50] [abbrv] incubator-unomi git commit: UNOMI-63 Use
ElasticSearch BulkProcessing to perform segment updates - Switch to
TransportClient to hopefully make load-balancing work better. This might have
a minor slowdown on standalone performance.
Posted by sh...@apache.org.
UNOMI-63 Use ElasticSearch BulkProcessing to perform segment updates
- Switch to TransportClient to hopefully make load-balancing work better. This might have a minor slowdown on standalone performance.
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/1075a02c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/1075a02c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/1075a02c
Branch: refs/heads/master
Commit: 1075a02cfeb6a4c59d935185a3f1d81fc5159f06
Parents: 661daee
Author: Serge Huber <sh...@apache.org>
Authored: Wed Nov 23 17:12:16 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Wed Nov 23 17:12:16 2016 +0100
----------------------------------------------------------------------
.../ElasticSearchPersistenceServiceImpl.java | 26 ++++++++++++++++----
1 file changed, 21 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1075a02c/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 733b3a9..794b03b 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -53,10 +53,12 @@ import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
+import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.DistanceUnit;
@@ -96,9 +98,7 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
+import java.net.*;
import java.nio.file.Paths;
import java.text.ParseException;
import java.text.SimpleDateFormat;
@@ -146,6 +146,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
public static final String ELASTICSEARCH_NETWORK_HOST = "network.host";
private Node node;
+ private Client nodeClient;
private Client client;
private BulkProcessor bulkProcessor;
private String clusterName;
@@ -356,15 +357,26 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
node = nodeBuilder().settings(settingsBuilder).node();
- client = node.client();
+ nodeClient = node.client();
+
logger.info("Waiting for ElasticSearch to start...");
- client.admin().cluster().prepareHealth()
+ nodeClient.admin().cluster().prepareHealth()
.setWaitForGreenStatus()
.get();
logger.info("Cluster status is GREEN");
+ try {
+ Settings transportSettings = Settings.settingsBuilder()
+ .put(CLUSTER_NAME, clusterName).build();
+ client = TransportClient.builder().settings(transportSettings).build()
+ .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(address), 9300));
+ } catch (UnknownHostException e) {
+ logger.error("Error resolving address " + address + " ElasticSearch transport client not connected, using internal client instead", e);
+ client = nodeClient;
+ }
+
// @todo is there a better way to detect index existence than to wait for it to startup ?
boolean indexExists = false;
int tries = 0;
@@ -570,6 +582,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
logger.error("Error waiting for bulk operations to flush !", e);
}
}
+ if (nodeClient != client) {
+ client.close();
+ }
+ nodeClient.close();
node.close();
return null;
}
[47/50] [abbrv] incubator-unomi git commit: - DMF-1133 Error when
displaying an empty list in MF : fixed issue for real this time :) - Some
configuration file cleanup - Changed inactive user purging from 30 days to
180 days
Posted by sh...@apache.org.
- DMF-1133 Error when displaying an empty list in MF : fixed issue for real this time :)
- Some configuration file cleanup
- Changed inactive user purging from 30 days to 180 days
Signed-off-by: Serge Huber <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/a5b7b156
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/a5b7b156
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/a5b7b156
Branch: refs/heads/master
Commit: a5b7b15650afd5b9086eedfce8fa406c8095d580
Parents: b812349
Author: Serge Huber <sh...@apache.org>
Authored: Fri Jan 6 11:15:28 2017 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Fri Jan 6 11:15:28 2017 +0100
----------------------------------------------------------------------
README.md | 56 +--
.../unomi/api/services/ClusterService.java | 1 -
kar/src/main/feature/feature.xml | 4 +-
package/pom.xml | 13 +-
persistence-elasticsearch/core/pom.xml | 20 -
.../ElasticSearchPersistenceServiceImpl.java | 397 +++++--------------
.../resources/OSGI-INF/blueprint/blueprint.xml | 21 +-
.../core/src/main/resources/elasticsearch.yml | 110 -----
.../core/src/main/resources/hazelcast.xml | 219 ----------
...g.apache.unomi.persistence.elasticsearch.cfg | 10 +-
.../persistence/spi/PersistenceService.java | 8 +
pom.xml | 2 -
services/pom.xml | 22 +
.../services/services/ClusterServiceImpl.java | 304 ++++++++++++++
.../resources/OSGI-INF/blueprint/blueprint.xml | 45 ++-
services/src/main/resources/hazelcast.xml | 219 ++++++++++
.../main/resources/org.apache.unomi.cluster.cfg | 20 +
.../resources/org.apache.unomi.services.cfg | 2 +-
src/site/markdown/clustering.md | 32 +-
src/site/markdown/configuration.md | 27 +-
20 files changed, 739 insertions(+), 793 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index b73efa4..fd6e1d7 100644
--- a/README.md
+++ b/README.md
@@ -116,16 +116,9 @@ with the following contents:
cluster.name=contextElasticSearch
index.name=context
- elasticSearchConfig=file:${karaf.etc}/elasticsearch.yml
And replace the cluster.name parameter here by your cluster name.
-
-You can also put an elasticsearch configuration file in $MY_KARAF_HOME/etc/elasticsearch.yml ,
-and put any standard Elasticsearch configuration options in this last file.
-
-If you want your context server to be a client only on a cluster of elasticsearch nodes, just set the node.data property
-to false.
-
+
Secured events configuration
---------------------------
@@ -260,23 +253,16 @@ servers on the same network, and enable the discovery protocol in $MY_KARAF_HOME
All nodes on the same network, sharing the same cluster name will be part of the same cluster.
-###Recommended configurations
-
-It is recommended to have one node dedicated to the context server, where the other nodes take care of the
-Elasticsearch persistence. The node dedicated to the context server will have node.data set to false.
-
#### 2 nodes configuration
One node dedicated to context server, 1 node for elasticsearch storage.
Node A :
- node.data=true
numberOfReplicas=0
monthlyIndex.numberOfReplicas=0
Node B :
- node.data=false
numberOfReplicas=0
monthlyIndex.numberOfReplicas=0
@@ -285,39 +271,19 @@ One node dedicated to context server, 2 nodes for elasticsearch storage with fau
Node A :
- node.data=false
numberOfReplicas=1
monthlyIndex.numberOfReplicas=1
Node B :
- node.data=true
numberOfReplicas=1
monthlyIndex.numberOfReplicas=1
Node C :
- node.data=true
numberOfReplicas=1
monthlyIndex.numberOfReplicas=1
-### Specific configuration
-If multicast is not allowed on your network, you'll need to switch to unicast protocol and manually configure the server IPs. This can be
-done by disabling the elasticsearch automatic discovery in $MY_KARAF_HOME/etc/org.apache.unomi.persistence.elasticsearch.cfg :
-
- discovery.zen.ping.multicast.enabled=false
-
-
-And then set the property discovery.zen.ping.unicast.hosts in $MY_KARAF_HOME/etc/elasticsearch.yml files :
-
-
- discovery.zen.ping.unicast.hosts: [\u2018192.168.0.1:9300', \u2018192.168.0.2:9300']
-
-
-More information and configuration options can be found at :
-[https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html)
-
-
JDK Selection on Mac OS X
-------------------------
@@ -415,20 +381,6 @@ Of course any ports listed here are the default ports configured in each server,
Step 2 : Adjust the Context Server IP filtering
-By default the Context Server limits to connections to port 9200 and 9300 to the following IP ranges
-
- - localhost
- - 127.0.0.1
- - ::1
- - the current subnet (i.e., 192.168.1.0-192.168.1.255)
-
-(this is done using a custom plugin for Elasticsearch, that you may find here :
-https://git-wip-us.apache.org/repos/asf/incubator-unomi/context-server/persistence-elasticsearch/plugins/security)
-
-You can adjust this setting by using the following setting in the $MY_KARAF_HOME/etc/elasticsearch.yml file :
-
- security.ipranges: localhost,127.0.0.1,::1,10.0.1.0-10.0.1.255
-
Step 3 : Follow industry recommended best practices for securing Elasticsearch
You may find more valuable recommendations here :
@@ -463,9 +415,3 @@ To upload the site to the Apache website, simply run after the above command has
This operation takes a little bit of time, so don't interrupt it even if you're waiting for a while for it to complete
(usually takes about 16 minutes !)
-
-Todo
-----
-
-- Look at possible integration with newsletter management systems such as MailChimp, for example to synchronize profile data with collected info.
-- Integrate with machine learning implementations such as Prediction.io or Apache Mahout
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/api/src/main/java/org/apache/unomi/api/services/ClusterService.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/unomi/api/services/ClusterService.java b/api/src/main/java/org/apache/unomi/api/services/ClusterService.java
index 37fcd37..b851b78 100644
--- a/api/src/main/java/org/apache/unomi/api/services/ClusterService.java
+++ b/api/src/main/java/org/apache/unomi/api/services/ClusterService.java
@@ -25,7 +25,6 @@ import java.util.List;
/**
* A service to access information about the context server's cluster.
*
- * TODO: rename to something less specific like ContextRuntimeService?
*/
public interface ClusterService {
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/kar/src/main/feature/feature.xml
----------------------------------------------------------------------
diff --git a/kar/src/main/feature/feature.xml b/kar/src/main/feature/feature.xml
index 8e6ddb2..fa3ef1f 100644
--- a/kar/src/main/feature/feature.xml
+++ b/kar/src/main/feature/feature.xml
@@ -29,10 +29,10 @@
<configfile finalname="/etc/org.apache.unomi.plugins.request.cfg">mvn:org.apache.unomi/unomi-plugins-request/${project.version}/cfg/requestcfg</configfile>
<configfile finalname="/etc/org.apache.unomi.services.cfg">mvn:org.apache.unomi/unomi-services/${project.version}/cfg/servicescfg</configfile>
<configfile finalname="/etc/org.apache.unomi.thirdparty.cfg">mvn:org.apache.unomi/unomi-services/${project.version}/cfg/thirdpartycfg</configfile>
+ <configfile finalname="/etc/org.apache.unomi.cluster.cfg">mvn:org.apache.unomi/unomi-services/${project.version}/cfg/clustercfg</configfile>
+ <configfile finalname="/etc/hazelcast.xml">mvn:org.apache.unomi/unomi-services/${project.version}/xml/hazelcastconfig</configfile>
<configfile finalname="/etc/org.apache.unomi.privacy.cfg">mvn:org.apache.unomi/cxs-privacy-extension-services/${project.version}/cfg/privacycfg</configfile>
<configfile finalname="/etc/org.apache.unomi.geonames.cfg">mvn:org.apache.unomi/cxs-geonames-services/${project.version}/cfg/geonamescfg</configfile>
- <configfile finalname="/etc/elasticsearch.yml">mvn:org.apache.unomi/unomi-persistence-elasticsearch-core/${project.version}/yml/elasticsearchconfig</configfile>
- <configfile finalname="/etc/hazelcast.xml">mvn:org.apache.unomi/unomi-persistence-elasticsearch-core/${project.version}/xml/hazelcastconfig</configfile>
<bundle start-level="75">mvn:commons-io/commons-io/2.4</bundle>
<bundle start-level="75">mvn:com.fasterxml.jackson.core/jackson-core/${version.jackson.core}</bundle>
<bundle start-level="75">mvn:com.fasterxml.jackson.core/jackson-databind/${version.jackson.core}</bundle>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/package/pom.xml
----------------------------------------------------------------------
diff --git a/package/pom.xml b/package/pom.xml
index 104723e..fd630fd 100644
--- a/package/pom.xml
+++ b/package/pom.xml
@@ -215,18 +215,7 @@
</artifactItem>
<artifactItem>
<groupId>org.apache.unomi</groupId>
- <artifactId>unomi-persistence-elasticsearch-core</artifactId>
- <version>${project.version}</version>
- <classifier>elasticsearchconfig</classifier>
- <type>yml</type>
- <outputDirectory>
- ${project.build.directory}/assembly/etc
- </outputDirectory>
- <destFileName>elasticsearch.yml</destFileName>
- </artifactItem>
- <artifactItem>
- <groupId>org.apache.unomi</groupId>
- <artifactId>unomi-persistence-elasticsearch-core</artifactId>
+ <artifactId>unomi-services</artifactId>
<version>${project.version}</version>
<classifier>hazelcastconfig</classifier>
<type>xml</type>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/persistence-elasticsearch/core/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/pom.xml b/persistence-elasticsearch/core/pom.xml
index f4b2a24..ad96cfb 100644
--- a/persistence-elasticsearch/core/pom.xml
+++ b/persistence-elasticsearch/core/pom.xml
@@ -54,16 +54,6 @@
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>org.apache.karaf.cellar</groupId>
- <artifactId>org.apache.karaf.cellar.core</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.karaf.cellar</groupId>
- <artifactId>org.apache.karaf.cellar.config</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>20.0</version>
@@ -255,16 +245,6 @@
<type>cfg</type>
<classifier>elasticsearchcfg</classifier>
</artifact>
- <artifact>
- <file>src/main/resources/elasticsearch.yml</file>
- <type>yml</type>
- <classifier>elasticsearchconfig</classifier>
- </artifact>
- <artifact>
- <file>src/main/resources/hazelcast.xml</file>
- <type>xml</type>
- <classifier>hazelcastconfig</classifier>
- </artifact>
</artifacts>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index f98854b..7abcbd5 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -18,13 +18,6 @@
package org.apache.unomi.persistence.elasticsearch;
import org.apache.commons.lang3.StringUtils;
-import org.apache.karaf.cellar.config.ClusterConfigurationEvent;
-import org.apache.karaf.cellar.config.Constants;
-import org.apache.karaf.cellar.core.*;
-import org.apache.karaf.cellar.core.control.SwitchStatus;
-import org.apache.karaf.cellar.core.event.EventProducer;
-import org.apache.karaf.cellar.core.event.EventType;
-import org.apache.unomi.api.ClusterNode;
import org.apache.unomi.api.Item;
import org.apache.unomi.api.PartialList;
import org.apache.unomi.api.TimestampedItem;
@@ -32,11 +25,12 @@ import org.apache.unomi.api.conditions.Condition;
import org.apache.unomi.api.query.DateRange;
import org.apache.unomi.api.query.IpRange;
import org.apache.unomi.api.query.NumericRange;
-import org.apache.unomi.api.services.ClusterService;
import org.apache.unomi.persistence.elasticsearch.conditions.*;
import org.apache.unomi.persistence.spi.CustomObjectMapper;
import org.apache.unomi.persistence.spi.PersistenceService;
import org.apache.unomi.persistence.spi.aggregate.*;
+import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
+import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
@@ -83,22 +77,13 @@ import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.osgi.framework.*;
-import org.osgi.service.cm.ConfigurationAdmin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.management.*;
-import javax.management.remote.JMXConnector;
-import javax.management.remote.JMXConnectorFactory;
-import javax.management.remote.JMXServiceURL;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.lang.management.ManagementFactory;
-import java.lang.management.OperatingSystemMXBean;
-import java.lang.management.RuntimeMXBean;
import java.net.InetAddress;
-import java.net.MalformedURLException;
import java.net.URL;
import java.net.UnknownHostException;
import java.text.ParseException;
@@ -108,7 +93,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@SuppressWarnings("rawtypes")
-public class ElasticSearchPersistenceServiceImpl implements PersistenceService, ClusterService, SynchronousBundleListener {
+public class ElasticSearchPersistenceServiceImpl implements PersistenceService, SynchronousBundleListener {
private static final Logger logger = LoggerFactory.getLogger(ElasticSearchPersistenceServiceImpl.class.getName());
@@ -116,18 +101,17 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
public static final String CONTEXTSERVER_PORT = "contextserver.port";
public static final String CONTEXTSERVER_SECURE_ADDRESS = "contextserver.secureAddress";
public static final String CONTEXTSERVER_SECURE_PORT = "contextserver.securePort";
+
public static final String NUMBER_OF_SHARDS = "number_of_shards";
public static final String NUMBER_OF_REPLICAS = "number_of_replicas";
public static final String CLUSTER_NAME = "cluster.name";
+
public static final String BULK_PROCESSOR_NAME = "bulkProcessor.name";
public static final String BULK_PROCESSOR_CONCURRENT_REQUESTS = "bulkProcessor.concurrentRequests";
public static final String BULK_PROCESSOR_BULK_ACTIONS = "bulkProcessor.bulkActions";
public static final String BULK_PROCESSOR_BULK_SIZE = "bulkProcessor.bulkSize";
public static final String BULK_PROCESSOR_FLUSH_INTERVAL = "bulkProcessor.flushInterval";
public static final String BULK_PROCESSOR_BACKOFF_POLICY = "bulkProcessor.backoffPolicy";
- public static final String KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION = "org.apache.unoni.nodes";
- public static final String KARAF_CLUSTER_CONFIGURATION_PUBLIC_ENDPOINTS = "publicEndpoints";
- public static final String KARAF_CLUSTER_CONFIGURATION_SECURE_ENDPOINTS = "secureEndpoints";
private Client client;
private BulkProcessor bulkProcessor;
@@ -137,19 +121,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private String monthlyIndexNumberOfReplicas;
private String numberOfShards;
private String numberOfReplicas;
- private String elasticSearchConfig = null;
private BundleContext bundleContext;
private Map<String, String> mappings = new HashMap<String, String>();
private ConditionEvaluatorDispatcher conditionEvaluatorDispatcher;
private ConditionESQueryBuilderDispatcher conditionESQueryBuilderDispatcher;
- private ClusterManager karafCellarClusterManager;
- private EventProducer karafCellarEventProducer;
- private GroupManager karafCellarGroupManager;
- private String karafCellarGroupName = Configurations.DEFAULT_GROUP_NAME;
- private ConfigurationAdmin osgiConfigurationAdmin;
- private String karafJMXUsername = "karaf";
- private String karafJMXPassword = "karaf";
- private int karafJMXPort = 1099;
private Map<String,String> indexNames;
private List<String> itemsMonthlyIndexed;
@@ -171,6 +146,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private String bulkProcessorFlushInterval = "5s";
private String bulkProcessorBackoffPolicy = "exponential";
+ private String minimalElasticSearchVersion = "5.0.0";
+ private String maximalElasticSearchVersion = "5.2.0";
+
private Map<String, Map<String, Map<String, Object>>> knownMappings = new HashMap<>();
public void setBundleContext(BundleContext bundleContext) {
@@ -233,10 +211,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
this.routingByType = routingByType;
}
- public void setElasticSearchConfig(String elasticSearchConfig) {
- this.elasticSearchConfig = elasticSearchConfig;
- }
-
public void setConditionEvaluatorDispatcher(ConditionEvaluatorDispatcher conditionEvaluatorDispatcher) {
this.conditionEvaluatorDispatcher = conditionEvaluatorDispatcher;
}
@@ -269,45 +243,21 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
this.bulkProcessorBackoffPolicy = bulkProcessorBackoffPolicy;
}
- public void setKarafCellarClusterManager(ClusterManager karafCellarClusterManager) {
- this.karafCellarClusterManager = karafCellarClusterManager;
+ public void setMinimalElasticSearchVersion(String minimalElasticSearchVersion) {
+ this.minimalElasticSearchVersion = minimalElasticSearchVersion;
}
- public void setKarafCellarEventProducer(EventProducer karafCellarEventProducer) {
- this.karafCellarEventProducer = karafCellarEventProducer;
+ public void setMaximalElasticSearchVersion(String maximalElasticSearchVersion) {
+ this.maximalElasticSearchVersion = maximalElasticSearchVersion;
}
- public void setKarafCellarGroupManager(GroupManager karafCellarGroupManager) {
- this.karafCellarGroupManager = karafCellarGroupManager;
- }
-
- public void setKarafCellarGroupName(String karafCellarGroupName) {
- this.karafCellarGroupName = karafCellarGroupName;
- }
-
- public void setOsgiConfigurationAdmin(ConfigurationAdmin osgiConfigurationAdmin) {
- this.osgiConfigurationAdmin = osgiConfigurationAdmin;
- }
-
- public void setKarafJMXUsername(String karafJMXUsername) {
- this.karafJMXUsername = karafJMXUsername;
- }
-
- public void setKarafJMXPassword(String karafJMXPassword) {
- this.karafJMXPassword = karafJMXPassword;
- }
-
- public void setKarafJMXPort(int karafJMXPort) {
- this.karafJMXPort = karafJMXPort;
- }
-
- public void start() {
+ public void start() throws Exception {
loadPredefinedMappings(bundleContext, false);
// on startup
new InClassLoaderExecute<Object>() {
- public Object execute(Object... args) {
+ public Object execute(Object... args) throws Exception {
logger.info("Connecting to ElasticSearch persistence backend using cluster name " + clusterName + " and index name " + indexName + "...");
address = System.getProperty(CONTEXTSERVER_ADDRESS, address);
@@ -315,51 +265,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
secureAddress = System.getProperty(CONTEXTSERVER_SECURE_ADDRESS, secureAddress);
securePort = System.getProperty(CONTEXTSERVER_SECURE_PORT, securePort);
- if (karafCellarEventProducer != null && karafCellarClusterManager != null) {
-
- boolean setupConfigOk = true;
- Group group = karafCellarGroupManager.findGroupByName(karafCellarGroupName);
- if (setupConfigOk && group == null) {
- logger.error("Cluster group " + karafCellarGroupName + " doesn't exist");
- setupConfigOk = false;
- }
-
- // check if the producer is ON
- if (setupConfigOk && karafCellarEventProducer.getSwitch().getStatus().equals(SwitchStatus.OFF)) {
- logger.error("Cluster event producer is OFF");
- setupConfigOk = false;
- }
-
- // check if the config pid is allowed
- if (setupConfigOk && !isClusterConfigPIDAllowed(group, Constants.CATEGORY, KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION, EventType.OUTBOUND)) {
- logger.error("Configuration PID " + KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION + " is blocked outbound for cluster group " + karafCellarGroupName);
- setupConfigOk = false;
- }
-
- if (setupConfigOk) {
- Map<String, Properties> configurations = karafCellarClusterManager.getMap(Constants.CONFIGURATION_MAP + Configurations.SEPARATOR + karafCellarGroupName);
- org.apache.karaf.cellar.core.Node thisKarafNode = karafCellarClusterManager.getNode();
- Properties karafCellarClusterNodeConfiguration = configurations.get(KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION);
- if (karafCellarClusterNodeConfiguration == null) {
- karafCellarClusterNodeConfiguration = new Properties();
- }
- String publicEndpointsPropValue = karafCellarClusterNodeConfiguration.getProperty(KARAF_CLUSTER_CONFIGURATION_PUBLIC_ENDPOINTS, thisKarafNode.getId() + "=" + address + ":" + port);
- String secureEndpointsPropValue = karafCellarClusterNodeConfiguration.getProperty(KARAF_CLUSTER_CONFIGURATION_SECURE_ENDPOINTS, thisKarafNode.getId() + "=" + secureAddress + ":" + securePort);
- String[] publicEndpointsArray = publicEndpointsPropValue.split(",");
- Set<String> publicEndpoints = new TreeSet<String>(Arrays.asList(publicEndpointsArray));
- String[] secureEndpointsArray = secureEndpointsPropValue.split(",");
- Set<String> secureEndpoints = new TreeSet<String>(Arrays.asList(secureEndpointsArray));
- publicEndpoints.add(thisKarafNode.getId() + "=" + address + ":" + port);
- secureEndpoints.add(thisKarafNode.getId() + "=" + secureAddress + ":" + securePort);
- karafCellarClusterNodeConfiguration.setProperty(KARAF_CLUSTER_CONFIGURATION_PUBLIC_ENDPOINTS, StringUtils.join(publicEndpoints, ","));
- karafCellarClusterNodeConfiguration.setProperty(KARAF_CLUSTER_CONFIGURATION_SECURE_ENDPOINTS, StringUtils.join(secureEndpoints, ","));
- configurations.put(KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION, karafCellarClusterNodeConfiguration);
- ClusterConfigurationEvent clusterConfigurationEvent = new ClusterConfigurationEvent(KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION);
- clusterConfigurationEvent.setSourceGroup(group);
- karafCellarEventProducer.produce(clusterConfigurationEvent);
- }
- }
-
bulkProcessorName = System.getProperty(BULK_PROCESSOR_NAME, bulkProcessorName);
bulkProcessorConcurrentRequests = System.getProperty(BULK_PROCESSOR_CONCURRENT_REQUESTS, bulkProcessorConcurrentRequests);
bulkProcessorBulkActions = System.getProperty(BULK_PROCESSOR_BULK_ACTIONS, bulkProcessorBulkActions);
@@ -373,7 +278,29 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
client = new PreBuiltTransportClient(transportSettings)
.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(address), 9300));
} catch (UnknownHostException e) {
- logger.error("Error resolving address " + address + " ElasticSearch transport client not connected, using internal client instead", e);
+ String message = "Error resolving address " + address + " ElasticSearch transport client not connected";
+ throw new Exception(message, e);
+ }
+
+ // let's now check the versions of all the nodes in the cluster, to make sure they are as expected.
+ try {
+ NodesInfoResponse nodesInfoResponse = client.admin().cluster().prepareNodesInfo()
+ .all().execute().get();
+
+ org.elasticsearch.Version minimalVersion = org.elasticsearch.Version.fromString(minimalElasticSearchVersion);
+ org.elasticsearch.Version maximalVersion = org.elasticsearch.Version.fromString(maximalElasticSearchVersion);
+ for (NodeInfo nodeInfo : nodesInfoResponse.getNodes()) {
+ org.elasticsearch.Version version = nodeInfo.getVersion();
+ if (version.before(minimalVersion) ||
+ version.equals(maximalVersion) ||
+ version.after(maximalVersion)) {
+ throw new Exception("ElasticSearch version on node " + nodeInfo.getHostname() + " is not within [" + minimalVersion + "," + maximalVersion + "), aborting startup !");
+ }
+ }
+ } catch (InterruptedException e) {
+ throw new Exception("Error checking ElasticSearch versions", e);
+ } catch (ExecutionException e) {
+ throw new Exception("Error checking ElasticSearch versions", e);
}
// @todo is there a better way to detect index existence than to wait for it to startup ?
@@ -433,7 +360,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
logger.info("Cluster status is GREEN");
- return null;
+ return true;
}
}.executeInClassLoader();
@@ -467,19 +394,19 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private void refreshExistingIndexNames() {
new InClassLoaderExecute<Boolean>() {
- protected Boolean execute(Object... args) {
+ protected Boolean execute(Object... args) throws Exception {
try {
logger.info("Refreshing existing indices list...");
IndicesStatsResponse indicesStatsResponse = client.admin().indices().prepareStats().all().execute().get();
existingIndexNames = new TreeSet<>(indicesStatsResponse.getIndices().keySet());
} catch (InterruptedException e) {
- logger.error("Error retrieving indices stats", e);
+ throw new Exception("Error retrieving indices stats", e);
} catch (ExecutionException e) {
- logger.error("Error retrieving indices stats", e);
+ throw new Exception("Error retrieving indices stats", e);
}
return true;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
public BulkProcessor getBulkProcessor() {
@@ -581,7 +508,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
return null;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
if (timer != null) {
timer.cancel();
@@ -706,7 +633,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
@Override
public <T extends Item> T load(final String itemId, final Date dateHint, final Class<T> clazz) {
return new InClassLoaderExecute<T>() {
- protected T execute(Object... args) {
+ protected T execute(Object... args) throws Exception {
try {
String itemType = (String) clazz.getField("ITEM_TYPE").get(null);
@@ -733,15 +660,14 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
}
} catch (IndexNotFoundException e) {
- logger.debug("No index found for itemType=" + clazz.getName() + " itemId=" + itemId, e);
+ throw new Exception("No index found for itemType=" + clazz.getName() + " itemId=" + itemId, e);
} catch (IllegalAccessException e) {
- logger.error("Error loading itemType=" + clazz.getName() + " itemId=" + itemId, e);
+ throw new Exception("Error loading itemType=" + clazz.getName() + " itemId=" + itemId, e);
} catch (Exception t) {
- logger.error("Error loading itemType=" + clazz.getName() + " itemId=" + itemId, t);
+ throw new Exception("Error loading itemType=" + clazz.getName() + " itemId=" + itemId, t);
}
- return null;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
@@ -754,7 +680,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
public boolean save(final Item item, final boolean useBatching) {
return new InClassLoaderExecute<Boolean>() {
- protected Boolean execute(Object... args) {
+ protected Boolean execute(Object... args) throws Exception {
try {
String source = CustomObjectMapper.getObjectMapper().writeValueAsString(item);
String itemType = item.getItemType();
@@ -794,11 +720,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
return true;
} catch (IOException e) {
- logger.error("Error saving item " + item, e);
+ throw new Exception("Error saving item " + item, e);
}
- return false;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
@@ -810,7 +735,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
@Override
public boolean update(final String itemId, final Date dateHint, final Class clazz, final Map source) {
return new InClassLoaderExecute<Boolean>() {
- protected Boolean execute(Object... args) {
+ protected Boolean execute(Object... args) throws Exception {
try {
String itemType = (String) clazz.getField("ITEM_TYPE").get(null);
@@ -827,21 +752,20 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
return true;
} catch (IndexNotFoundException e) {
- logger.debug("No index found for itemType=" + clazz.getName() + "itemId=" + itemId, e);
+ throw new Exception("No index found for itemType=" + clazz.getName() + "itemId=" + itemId, e);
} catch (NoSuchFieldException e) {
- logger.error("Error updating item " + itemId, e);
+ throw new Exception("Error updating item " + itemId, e);
} catch (IllegalAccessException e) {
- logger.error("Error updating item " + itemId, e);
+ throw new Exception("Error updating item " + itemId, e);
}
- return false;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
@Override
public boolean updateWithScript(final String itemId, final Date dateHint, final Class<?> clazz, final String script, final Map<String, Object> scriptParams) {
return new InClassLoaderExecute<Boolean>() {
- protected Boolean execute(Object... args) {
+ protected Boolean execute(Object... args) throws Exception {
try {
String itemType = (String) clazz.getField("ITEM_TYPE").get(null);
@@ -859,21 +783,20 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
return true;
} catch (IndexNotFoundException e) {
- logger.debug("No index found for itemType=" + clazz.getName() + "itemId=" + itemId, e);
+ throw new Exception("No index found for itemType=" + clazz.getName() + "itemId=" + itemId, e);
} catch (NoSuchFieldException e) {
- logger.error("Error updating item " + itemId, e);
+ throw new Exception("Error updating item " + itemId, e);
} catch (IllegalAccessException e) {
- logger.error("Error updating item " + itemId, e);
+ throw new Exception("Error updating item " + itemId, e);
}
- return false;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
@Override
public <T extends Item> boolean remove(final String itemId, final Class<T> clazz) {
return new InClassLoaderExecute<Boolean>() {
- protected Boolean execute(Object... args) {
+ protected Boolean execute(Object... args) throws Exception {
//Index the query = register it in the percolator
try {
String itemType = (String) clazz.getField("ITEM_TYPE").get(null);
@@ -882,16 +805,15 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
.execute().actionGet();
return true;
} catch (Exception e) {
- logger.error("Cannot remove", e);
+ throw new Exception("Cannot remove", e);
}
- return false;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
public <T extends Item> boolean removeByQuery(final Condition query, final Class<T> clazz) {
return new InClassLoaderExecute<Boolean>() {
- protected Boolean execute(Object... args) {
+ protected Boolean execute(Object... args) throws Exception {
try {
String itemType = (String) clazz.getField("ITEM_TYPE").get(null);
@@ -932,11 +854,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
return true;
} catch (Exception e) {
- logger.error("Cannot remove by query", e);
+ throw new Exception("Cannot remove by query", e);
}
- return false;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
public boolean createIndex(final String indexName) {
@@ -956,7 +877,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
return !indexExists;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
public boolean removeIndex(final String indexName) {
@@ -970,7 +891,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
return indexExists;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
private void internalCreateIndex(String indexName, Map<String,String> mappings) {
@@ -1029,7 +950,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
public Map<String, Map<String, Object>> getPropertiesMapping(final String itemType) {
return new InClassLoaderExecute<Map<String, Map<String, Object>>>() {
@SuppressWarnings("unchecked")
- protected Map<String, Map<String, Object>> execute(Object... args) {
+ protected Map<String, Map<String, Object>> execute(Object... args) throws Exception {
GetMappingsResponse getMappingsResponse = client.admin().indices().prepareGetMappings().setTypes(itemType).execute().actionGet();
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = getMappingsResponse.getMappings();
Map<String, Map<String, Object>> propertyMap = new HashMap<>();
@@ -1054,11 +975,11 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
}
} catch (IOException e) {
- logger.error("Cannot get mapping", e);
+ throw new Exception("Cannot get mapping", e);
}
return propertyMap;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
public Map<String, Object> getPropertyMapping(String property, String itemType) {
@@ -1089,6 +1010,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private String getPropertyNameWithData(String name, String itemType) {
Map<String,Object> propertyMapping = getPropertyMapping(name,itemType);
+ if (propertyMapping == null) {
+ return null;
+ }
if (propertyMapping != null
&& "text".equals(propertyMapping.get("type"))
&& propertyMapping.containsKey("fields")
@@ -1100,7 +1024,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
public boolean saveQuery(final String queryName, final String query) {
return new InClassLoaderExecute<Boolean>() {
- protected Boolean execute(Object... args) {
+ protected Boolean execute(Object... args) throws Exception {
//Index the query = register it in the percolator
try {
logger.info("Saving query : " + queryName);
@@ -1110,11 +1034,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
.execute().actionGet();
return true;
} catch (Exception e) {
- logger.error("Cannot save query", e);
+ throw new Exception("Cannot save query", e);
}
- return false;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
@Override
@@ -1129,7 +1052,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
@Override
public boolean removeQuery(final String queryName) {
return new InClassLoaderExecute<Boolean>() {
- protected Boolean execute(Object... args) {
+ protected Boolean execute(Object... args) throws Exception {
//Index the query = register it in the percolator
try {
client.prepareDelete(indexName, ".percolator", queryName)
@@ -1137,11 +1060,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
.execute().actionGet();
return true;
} catch (Exception e) {
- logger.error("Cannot delete query", e);
+ throw new Exception("Cannot delete query", e);
}
- return false;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
@Override
@@ -1238,14 +1160,14 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
.actionGet();
return response.getHits().getTotalHits();
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
private <T extends Item> PartialList<T> query(final QueryBuilder query, final String sortBy, final Class<T> clazz, final int offset, final int size, final String[] routing, final String scrollTimeValidity) {
return new InClassLoaderExecute<PartialList<T>>() {
@Override
- protected PartialList<T> execute(Object... args) {
+ protected PartialList<T> execute(Object... args) throws Exception {
List<T> results = new ArrayList<T>();
String scrollIdentifier = null;
long totalHits = 0;
@@ -1348,7 +1270,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
}
} catch (Exception t) {
- logger.error("Error loading itemType=" + clazz.getName() + " query=" + query + " sortBy=" + sortBy, t);
+ throw new Exception("Error loading itemType=" + clazz.getName() + " query=" + query + " sortBy=" + sortBy, t);
}
PartialList<T> result = new PartialList<T>(results, offset, size, totalHits);
@@ -1358,7 +1280,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
return result;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
@Override
@@ -1366,7 +1288,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
return new InClassLoaderExecute<PartialList<T>>() {
@Override
- protected PartialList<T> execute(Object... args) {
+ protected PartialList<T> execute(Object... args) throws Exception {
List<T> results = new ArrayList<T>();
long totalHits = 0;
try {
@@ -1391,11 +1313,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
return result;
} catch (Exception t) {
- logger.error("Error continuing scrolling query for itemType=" + clazz.getName() + " scrollIdentifier=" + scrollIdentifier + " scrollTimeValidity=" + scrollTimeValidity, t);
+ throw new Exception("Error continuing scrolling query for itemType=" + clazz.getName() + " scrollIdentifier=" + scrollIdentifier + " scrollTimeValidity=" + scrollTimeValidity, t);
}
- return null;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
@Override
@@ -1520,7 +1441,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
return results;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
private <T extends Item> String getItemType(Class<T> clazz) {
@@ -1544,105 +1465,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
- @Override
- public List<ClusterNode> getClusterNodes() {
- return new InClassLoaderExecute<List<ClusterNode>>() {
-
- @Override
- protected List<ClusterNode> execute(Object... args) {
- Map<String, ClusterNode> clusterNodes = new LinkedHashMap<String, ClusterNode>();
-
- Set<org.apache.karaf.cellar.core.Node> karafCellarNodes = karafCellarClusterManager.listNodes();
- org.apache.karaf.cellar.core.Node thisKarafNode = karafCellarClusterManager.getNode();
- Map<String, Properties> clusterConfigurations = karafCellarClusterManager.getMap(Constants.CONFIGURATION_MAP + Configurations.SEPARATOR + karafCellarGroupName);
- Properties karafCellarClusterNodeConfiguration = clusterConfigurations.get(KARAF_CELLAR_CLUSTER_NODE_CONFIGURATION);
- Map<String, String> publicNodeEndpoints = new TreeMap<>();
- Map<String, String> secureNodeEndpoints = new TreeMap<>();
- if (karafCellarClusterNodeConfiguration != null) {
- String publicEndpointsPropValue = karafCellarClusterNodeConfiguration.getProperty(KARAF_CLUSTER_CONFIGURATION_PUBLIC_ENDPOINTS, thisKarafNode.getId() + "=" + address + ":" + port);
- String secureEndpointsPropValue = karafCellarClusterNodeConfiguration.getProperty(KARAF_CLUSTER_CONFIGURATION_SECURE_ENDPOINTS, thisKarafNode.getId() + "=" + secureAddress + ":" + securePort);
- String[] publicEndpointsArray = publicEndpointsPropValue.split(",");
- Set<String> publicEndpoints = new TreeSet<String>(Arrays.asList(publicEndpointsArray));
- for (String endpoint : publicEndpoints) {
- String[] endpointParts = endpoint.split("=");
- publicNodeEndpoints.put(endpointParts[0], endpointParts[1]);
- }
- String[] secureEndpointsArray = secureEndpointsPropValue.split(",");
- Set<String> secureEndpoints = new TreeSet<String>(Arrays.asList(secureEndpointsArray));
- for (String endpoint : secureEndpoints) {
- String[] endpointParts = endpoint.split("=");
- secureNodeEndpoints.put(endpointParts[0], endpointParts[1]);
- }
- }
- for (org.apache.karaf.cellar.core.Node karafCellarNode : karafCellarNodes) {
- ClusterNode clusterNode = new ClusterNode();
- clusterNode.setHostName(karafCellarNode.getHost());
- String publicEndpoint = publicNodeEndpoints.get(karafCellarNode.getId());
- if (publicEndpoint != null) {
- String[] publicEndpointParts = publicEndpoint.split(":");
- clusterNode.setHostAddress(publicEndpointParts[0]);
- clusterNode.setPublicPort(Integer.parseInt(publicEndpointParts[1]));
- }
- String secureEndpoint = secureNodeEndpoints.get(karafCellarNode.getId());
- if (secureEndpoint != null) {
- String[] secureEndpointParts = secureEndpoint.split(":");
- clusterNode.setSecureHostAddress(secureEndpointParts[0]);
- clusterNode.setSecurePort(Integer.parseInt(secureEndpointParts[1]));
- clusterNode.setMaster(false);
- clusterNode.setData(false);
- }
- try {
- // now let's connect to remote JMX service to retrieve information from the runtime and operating system MX beans
- JMXServiceURL url = new JMXServiceURL("service:jmx:rmi:///jndi/rmi://"+karafCellarNode.getHost() + ":"+karafJMXPort+"/karaf-root");
- Map<String,Object> environment=new HashMap<String,Object>();
- if (karafJMXUsername != null && karafJMXPassword != null) {
- environment.put(JMXConnector.CREDENTIALS,new String[]{karafJMXUsername,karafJMXPassword});
- }
- JMXConnector jmxc = JMXConnectorFactory.connect(url, environment);
- MBeanServerConnection mbsc = jmxc.getMBeanServerConnection();
- final RuntimeMXBean remoteRuntime = ManagementFactory.newPlatformMXBeanProxy(mbsc, ManagementFactory.RUNTIME_MXBEAN_NAME, RuntimeMXBean.class);
- clusterNode.setUptime(remoteRuntime.getUptime());
- ObjectName operatingSystemMXBeanName = new ObjectName(ManagementFactory.OPERATING_SYSTEM_MXBEAN_NAME);
- Double processCpuLoad = null;
- Double systemCpuLoad = null;
- try {
- processCpuLoad = (Double) mbsc.getAttribute(operatingSystemMXBeanName, "ProcessCpuLoad");
- } catch (MBeanException e) {
- e.printStackTrace();
- } catch (AttributeNotFoundException e) {
- e.printStackTrace();
- }
- try {
- systemCpuLoad = (Double) mbsc.getAttribute(operatingSystemMXBeanName, "SystemCpuLoad");
- } catch (MBeanException e) {
- e.printStackTrace();
- } catch (AttributeNotFoundException e) {
- e.printStackTrace();
- }
- final OperatingSystemMXBean remoteOperatingSystemMXBean = ManagementFactory.newPlatformMXBeanProxy(mbsc, ManagementFactory.OPERATING_SYSTEM_MXBEAN_NAME, OperatingSystemMXBean.class);
- clusterNode.setLoadAverage(new double[] { remoteOperatingSystemMXBean.getSystemLoadAverage()});
- if (systemCpuLoad != null) {
- clusterNode.setCpuLoad(systemCpuLoad);
- }
-
- } catch (MalformedURLException e) {
- logger.error("Error connecting to remote JMX server", e);
- } catch (IOException e) {
- logger.error("Error retrieving remote JMX data", e);
- } catch (MalformedObjectNameException e) {
- logger.error("Error retrieving remote JMX data", e);
- } catch (InstanceNotFoundException e) {
- logger.error("Error retrieving remote JMX data", e);
- } catch (ReflectionException e) {
- logger.error("Error retrieving remote JMX data", e);
- }
- clusterNodes.put(karafCellarNode.getId(), clusterNode);
- }
-
- return new ArrayList<ClusterNode>(clusterNodes.values());
- }
- }.executeInClassLoader();
- }
@Override
public void refresh() {
@@ -1654,7 +1476,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
client.admin().indices().refresh(Requests.refreshRequest()).actionGet();
return true;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
@@ -1663,7 +1485,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
public void purge(final Date date) {
new InClassLoaderExecute<Object>() {
@Override
- protected Object execute(Object... args) {
+ protected Object execute(Object... args) throws Exception {
IndicesStatsResponse statsResponse = client.admin().indices().prepareStats(indexName + "-*")
.setIndexing(false)
.setGet(false)
@@ -1689,7 +1511,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
toDelete.add(currentIndexName);
}
} catch (ParseException e) {
- logger.error("Cannot parse index name " + currentIndexName, e);
+ throw new Exception("Cannot parse index name " + currentIndexName, e);
}
}
}
@@ -1698,7 +1520,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
return null;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
@Override
@@ -1743,7 +1565,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
return null;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
@Override
@@ -1794,7 +1616,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
return results;
}
- }.executeInClassLoader();
+ }.catchingExecuteInClassLoader(true);
}
private String getIndexNameForQuery(String itemType) {
@@ -1804,9 +1626,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
public abstract static class InClassLoaderExecute<T> {
- protected abstract T execute(Object... args);
+ protected abstract T execute(Object... args) throws Exception;
- public T executeInClassLoader(Object... args) {
+ public T executeInClassLoader(Object... args) throws Exception {
ClassLoader tccl = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
@@ -1815,6 +1637,15 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
Thread.currentThread().setContextClassLoader(tccl);
}
}
+
+ public T catchingExecuteInClassLoader( boolean logError, Object... args) {
+ try {
+ return executeInClassLoader(args);
+ } catch (Exception e) {
+ logger.error("Error while executing in class loader", e);
+ }
+ return null;
+ }
}
private String getConfig(Map<String,String> settings, String key,
@@ -1825,21 +1656,5 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
return defaultValue;
}
- /**
- * Check if a configuration is allowed.
- *
- * @param group the cluster group.
- * @param category the configuration category constant.
- * @param pid the configuration PID.
- * @param type the cluster event type.
- * @return true if the cluster event type is allowed, false else.
- */
- public boolean isClusterConfigPIDAllowed(Group group, String category, String pid, EventType type) {
- CellarSupport support = new CellarSupport();
- support.setClusterManager(this.karafCellarClusterManager);
- support.setGroupManager(this.karafCellarGroupManager);
- support.setConfigurationAdmin(this.osgiConfigurationAdmin);
- return support.isAllowed(group, category, pid, type);
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index c929009..1135ef5 100644
--- a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -32,7 +32,6 @@
<cm:default-properties>
<cm:property name="cluster.name" value="contextElasticSearch"/>
<cm:property name="index.name" value="context"/>
- <cm:property name="elasticSearchConfig" value="file:${karaf.etc}/elasticsearch.yml"/>
<cm:property name="numberOfShards" value="5"/>
<cm:property name="numberOfReplicas" value="0"/>
<cm:property name="monthlyIndex.numberOfShards" value="3"/>
@@ -46,10 +45,9 @@
<cm:property name="bulkProcessor.flushInterval" value="5s" />
<cm:property name="bulkProcessor.backoffPolicy" value="exponential" />
- <cm:property name="cluster.group" value="default" />
- <cm:property name="cluster.jmxUsername" value="karaf" />
- <cm:property name="cluster.jmxPassword" value="karaf" />
- <cm:property name="cluster.jmxPort" value="1099" />
+ <cm:property name="minimalElasticSearchVersion" value="5.0.0" />
+ <cm:property name="maximalElasticSearchVersion" value="5.2.0" />
+
</cm:default-properties>
</cm:property-placeholder>
@@ -71,7 +69,6 @@
<service id="elasticSearchPersistenceService" ref="elasticSearchPersistenceServiceImpl">
<interfaces>
<value>org.apache.unomi.persistence.spi.PersistenceService</value>
- <value>org.apache.unomi.api.services.ClusterService</value>
</interfaces>
</service>
@@ -101,7 +98,6 @@
<property name="secureAddress" value="${web.contextserver.secureAddress}"/>
<property name="securePort" value="${web.contextserver.securePort}"/>
<property name="defaultQueryLimit" value="${es.defaultQueryLimit}"/>
- <property name="elasticSearchConfig" value="${es.elasticSearchConfig}"/>
<property name="itemsMonthlyIndexed">
<list>
<value>event</value>
@@ -123,14 +119,9 @@
<property name="bulkProcessorBulkSize" value="${es.bulkProcessor.bulkSize}" />
<property name="bulkProcessorFlushInterval" value="${es.bulkProcessor.flushInterval}" />
<property name="bulkProcessorBackoffPolicy" value="${es.bulkProcessor.backoffPolicy}" />
- <property name="karafCellarClusterManager" ref="karafCellarClusterManager" />
- <property name="karafCellarEventProducer" ref="karafCellarEventProducer" />
- <property name="karafCellarGroupManager" ref="karafCellarGroupManager" />
- <property name="karafCellarGroupName" value="${es.cluster.group}" />
- <property name="osgiConfigurationAdmin" ref="osgiConfigurationAdmin" />
- <property name="karafJMXUsername" value="${es.cluster.jmxUsername}" />
- <property name="karafJMXPassword" value="${es.cluster.jmxPassword}" />
- <property name="karafJMXPort" value="${es.cluster.jmxPort}" />
+
+ <property name="minimalElasticSearchVersion" value="${es.minimalElasticSearchVersion}" />
+ <property name="maximalElasticSearchVersion" value="${es.maximalElasticSearchVersion}" />
</bean>
<!-- We use a listener here because using the list directly for listening to proxies coming from the same bundle didn't seem to work -->
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/persistence-elasticsearch/core/src/main/resources/elasticsearch.yml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/elasticsearch.yml b/persistence-elasticsearch/core/src/main/resources/elasticsearch.yml
deleted file mode 100644
index fe0e52f..0000000
--- a/persistence-elasticsearch/core/src/main/resources/elasticsearch.yml
+++ /dev/null
@@ -1,110 +0,0 @@
-# ======================== Elasticsearch Configuration =========================
-#
-# NOTE: Elasticsearch comes with reasonable defaults for most settings.
-# Before you set out to tweak and tune the configuration, make sure you
-# understand what are you trying to accomplish and the consequences.
-#
-# The primary way of configuring a node is via this file. This template lists
-# the most important settings you may want to configure for a production cluster.
-#
-# Please see the documentation for further information on configuration options:
-# <http://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration.html>
-#
-# ---------------------------------- Cluster -----------------------------------
-#
-# Use a descriptive name for your cluster:
-#
-# cluster.name: my-application
-#
-# ------------------------------------ Node ------------------------------------
-#
-# Use a descriptive name for the node:
-#
-# node.name: node-1
-#
-# Add custom attributes to the node:
-#
-# node.rack: r1
-#
-# ----------------------------------- Paths ------------------------------------
-#
-# Path to directory where to store the data (separate multiple locations by comma):
-#
-# path.data: /path/to/data
-#
-# Path to log files:
-#
-# path.logs: /path/to/logs
-#
-# ----------------------------------- Memory -----------------------------------
-#
-# Lock the memory on startup:
-#
-# bootstrap.mlockall: true
-#
-# Make sure that the `ES_HEAP_SIZE` environment variable is set to about half the memory
-# available on the system and that the owner of the process is allowed to use this limit.
-#
-# Elasticsearch performs poorly when the system is swapping the memory.
-#
-# ---------------------------------- Network -----------------------------------
-#
-# Set the bind address to a specific IP (IPv4 or IPv6):
-#
-# network.host: 192.168.0.1
-#
-# Set a custom port for HTTP:
-#
-# http.port: 9200
-#
-# For more information, see the documentation at:
-# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html>
-#
-# --------------------------------- Discovery ----------------------------------
-#
-# Pass an initial list of hosts to perform discovery when new node is started:
-# The default list of hosts is ["127.0.0.1", "[::1]"]
-#
-# discovery.zen.ping.unicast.hosts: ["host1", "host2"]
-#
-# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
-#
-# discovery.zen.minimum_master_nodes: 3
-#
-# For more information, see the documentation at:
-# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html>
-#
-# ---------------------------------- Gateway -----------------------------------
-#
-# Block initial recovery after a full cluster restart until N nodes are started:
-#
-# gateway.recover_after_nodes: 3
-#
-# For more information, see the documentation at:
-# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-gateway.html>
-#
-# ---------------------------------- Various -----------------------------------
-#
-# Disable starting multiple nodes on a single system:
-#
-# node.max_local_storage_nodes: 1
-#
-# Require explicit names when deleting indices:
-#
-# action.destructive_requires_name: true
-#
-# --------------------------- Apache Unomi specific ---------------------------
-#
-threadpool.index.queue_size: 1000
-script.engine.groovy.inline.update: on
-index.percolator.allow_unmapped_fields: true
-
-# Require explicit index creation
-action.auto_create_index: false
-
-# Protect against accidental close/delete operations
-# on all indices. You can still close/delete individual
-# indices
-#action.disable_close_all_indices: true
-#action.disable_delete_all_indices: true
-#action.disable_shutdown: true
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/persistence-elasticsearch/core/src/main/resources/hazelcast.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/hazelcast.xml b/persistence-elasticsearch/core/src/main/resources/hazelcast.xml
deleted file mode 100644
index 0fc6f5d..0000000
--- a/persistence-elasticsearch/core/src/main/resources/hazelcast.xml
+++ /dev/null
@@ -1,219 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-<hazelcast xsi:schemaLocation="http://www.hazelcast.com/schema/config hazelcast-config-3.2.xsd"
- xmlns="http://www.hazelcast.com/schema/config"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <group>
- <name>cellar</name>
- <password>pass</password>
- </group>
- <management-center enabled="false">http://localhost:8080/mancenter</management-center>
- <network>
- <port auto-increment="true" port-count="100">5701</port>
- <outbound-ports>
- <!--
- Allowed port range when connecting to other nodes.
- 0 or * means use system provided port.
- -->
- <ports>0</ports>
- </outbound-ports>
- <join>
- <multicast enabled="false">
- <multicast-group>224.2.2.3</multicast-group>
- <multicast-port>54327</multicast-port>
- </multicast>
- <tcp-ip enabled="true">
- <interface>127.0.0.1</interface>
- </tcp-ip>
- <aws enabled="false">
- <access-key>my-access-key</access-key>
- <secret-key>my-secret-key</secret-key>
- <!--optional, default is us-east-1 -->
- <region>us-west-1</region>
- <!--optional, default is ec2.amazonaws.com. If set, region shouldn't be set as it will override this property -->
- <host-header>ec2.amazonaws.com</host-header>
- <!-- optional, only instances belonging to this group will be discovered, default will try all running instances -->
- <security-group-name>hazelcast-sg</security-group-name>
- <tag-key>type</tag-key>
- <tag-value>hz-nodes</tag-value>
- </aws>
- </join>
- <interfaces enabled="false">
- <interface>10.10.1.*</interface>
- </interfaces>
- <ssl enabled="false"/>
- <socket-interceptor enabled="false"/>
- <symmetric-encryption enabled="false">
- <!--
- encryption algorithm such as
- DES/ECB/PKCS5Padding,
- PBEWithMD5AndDES,
- AES/CBC/PKCS5Padding,
- Blowfish,
- DESede
- -->
- <algorithm>PBEWithMD5AndDES</algorithm>
- <!-- salt value to use when generating the secret key -->
- <salt>thesalt</salt>
- <!-- pass phrase to use when generating the secret key -->
- <password>thepass</password>
- <!-- iteration count to use when generating the secret key -->
- <iteration-count>19</iteration-count>
- </symmetric-encryption>
- </network>
- <partition-group enabled="false"/>
- <executor-service>
- <pool-size>16</pool-size>
- <!-- Queue capacity. 0 means Integer.MAX_VALUE -->
- <queue-capacity>0</queue-capacity>
- </executor-service>
- <queue name="default">
- <!--
- Maximum size of the queue. When a JVM's local queue size reaches the maximum,
- all put/offer operations will get blocked until the queue size
- of the JVM goes down below the maximum.
- Any integer between 0 and Integer.MAX_VALUE. 0 means
- Integer.MAX_VALUE. Default is 0.
- -->
- <max-size>0</max-size>
- <!--
- Number of backups. If 1 is set as the backup-count for example,
- then all entries of the map will be copied to another JVM for
- fail-safety. 0 means no backup.
- -->
- <backup-count>1</backup-count>
- <!--
- Number of async backups. 0 means no backup.
- -->
- <async-backup-count>0</async-backup-count>
- <empty-queue-ttl>-1</empty-queue-ttl>
- </queue>
-
- <map name="default">
- <!--
- Data type that will be used for storing recordMap.
- Possible values:
- BINARY (default): keys and values will be stored as binary data
- OBJECT : values will be stored in their object forms
- OFFHEAP : values will be stored in non-heap region of JVM
- -->
- <in-memory-format>BINARY</in-memory-format>
- <!--
- Number of backups. If 1 is set as the backup-count for example,
- then all entries of the map will be copied to another JVM for
- fail-safety. 0 means no backup.
- -->
- <backup-count>1</backup-count>
- <!--
- Number of async backups. 0 means no backup.
- -->
- <async-backup-count>0</async-backup-count>
- <!--
- Maximum number of seconds for each entry to stay in the map. Entries that are
- older than <time-to-live-seconds> and not updated for <time-to-live-seconds>
- will get automatically evicted from the map.
- Any integer between 0 and Integer.MAX_VALUE. 0 means infinite. Default is 0.
- -->
- <time-to-live-seconds>0</time-to-live-seconds>
- <!--
- Maximum number of seconds for each entry to stay idle in the map. Entries that are
- idle(not touched) for more than <max-idle-seconds> will get
- automatically evicted from the map. Entry is touched if get, put or containsKey is called.
- Any integer between 0 and Integer.MAX_VALUE. 0 means infinite. Default is 0.
- -->
- <max-idle-seconds>0</max-idle-seconds>
- <!--
- Valid values are:
- NONE (no eviction),
- LRU (Least Recently Used),
- LFU (Least Frequently Used).
- NONE is the default.
- -->
- <eviction-policy>NONE</eviction-policy>
- <!--
- Maximum size of the map. When max size is reached,
- map is evicted based on the policy defined.
- Any integer between 0 and Integer.MAX_VALUE. 0 means
- Integer.MAX_VALUE. Default is 0.
- -->
- <max-size policy="PER_NODE">0</max-size>
- <!--
- When max. size is reached, specified percentage of
- the map will be evicted. Any integer between 0 and 100.
- If 25 is set for example, 25% of the entries will
- get evicted.
- -->
- <eviction-percentage>25</eviction-percentage>
- <!--
- While recovering from split-brain (network partitioning),
- map entries in the small cluster will merge into the bigger cluster
- based on the policy set here. When an entry merge into the
- cluster, there might an existing entry with the same key already.
- Values of these entries might be different for that same key.
- Which value should be set for the key? Conflict is resolved by
- the policy set here. Default policy is PutIfAbsentMapMergePolicy
-
- There are built-in merge policies such as
- com.hazelcast.map.merge.PassThroughMergePolicy; entry will be added if there is no existing entry for the key.
- com.hazelcast.map.merge.PutIfAbsentMapMergePolicy ; entry will be added if the merging entry doesn't exist in the cluster.
- com.hazelcast.map.merge.HigherHitsMapMergePolicy ; entry with the higher hits wins.
- com.hazelcast.map.merge.LatestUpdateMapMergePolicy ; entry with the latest update wins.
- -->
- <merge-policy>com.hazelcast.map.merge.PassThroughMergePolicy</merge-policy>
- </map>
-
- <multimap name="default">
- <backup-count>1</backup-count>
- <value-collection-type>SET</value-collection-type>
- </multimap>
-
- <multimap name="default">
- <backup-count>1</backup-count>
- <value-collection-type>SET</value-collection-type>
- </multimap>
-
- <list name="default">
- <backup-count>1</backup-count>
- </list>
-
- <set name="default">
- <backup-count>1</backup-count>
- </set>
-
- <jobtracker name="default">
- <max-thread-size>0</max-thread-size>
- <!-- Queue size 0 means number of partitions * 2 -->
- <queue-size>0</queue-size>
- <retry-count>0</retry-count>
- <chunk-size>1000</chunk-size>
- <communicate-stats>true</communicate-stats>
- <topology-changed-strategy>CANCEL_RUNNING_OPERATION</topology-changed-strategy>
- </jobtracker>
-
- <semaphore name="default">
- <initial-permits>0</initial-permits>
- <backup-count>1</backup-count>
- <async-backup-count>0</async-backup-count>
- </semaphore>
-
- <serialization>
- <portable-version>0</portable-version>
- </serialization>
-
- <services enable-defaults="true" />
-</hazelcast>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
index ddcbed5..55a24ea 100644
--- a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
+++ b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
@@ -17,14 +17,10 @@
cluster.name=contextElasticSearch
index.name=context
-elasticSearchConfig=file:${karaf.etc}/elasticsearch.yml
monthlyIndex.numberOfShards=3
monthlyIndex.numberOfReplicas=0
numberOfShards=5
numberOfReplicas=0
-node.data=true
-discovery.zen.ping.multicast.enabled=false
-#discovery.zen.ping.unicast.hosts=["192.168.0.1:9300", "192.168.0.2:9300"]
defaultQueryLimit=10
# The following settings control the behavior of the BulkProcessor API. You can find more information about these
@@ -37,7 +33,5 @@ bulkProcessor.bulkSize=5MB
bulkProcessor.flushInterval=5s
bulkProcessor.backoffPolicy=exponential
-cluster.group=default
-cluster.jmxUsername=karaf
-cluster.jmxPassword=karaf
-cluster.jmxPort=1099
\ No newline at end of file
+minimalElasticSearchVersion=5.0.0
+maximalElasticSearchVersion=5.2.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
----------------------------------------------------------------------
diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
index 75c0a3b..a6b175f 100644
--- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
+++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
@@ -452,4 +452,12 @@ public interface PersistenceService {
* @return {@code true} if the operation was successful, {@code false} otherwise
*/
boolean removeIndex(final String indexName);
+
+ /**
+ * Removes all data associated with the provided scope.
+ *
+ * @param scope the scope for which we want to remove data
+ */
+ void purge(final String scope);
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1aba4e5..6de2441 100644
--- a/pom.xml
+++ b/pom.xml
@@ -427,8 +427,6 @@
<exclude>**/src/main/webapp/WEB-INF/beans.xml</exclude>
<!-- Web application robots.txt file -->
<exclude>**/src/main/webapp/robots.txt</exclude>
- <!-- Elastic search configuration files with (mostly) default configuration -->
- <exclude>**/src/main/resources/elasticsearch.yml</exclude>
<!-- ignore generated log files -->
<exclude>**/*.log</exclude>
</excludes>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a5b7b156/services/pom.xml
----------------------------------------------------------------------
diff --git a/services/pom.xml b/services/pom.xml
index 7a58cd7..5188aea 100644
--- a/services/pom.xml
+++ b/services/pom.xml
@@ -112,6 +112,16 @@
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.karaf.cellar</groupId>
+ <artifactId>org.apache.karaf.cellar.core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.karaf.cellar</groupId>
+ <artifactId>org.apache.karaf.cellar.config</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
@@ -156,6 +166,18 @@
<type>cfg</type>
<classifier>thirdpartycfg</classifier>
</artifact>
+ <artifact>
+ <file>
+ src/main/resources/org.apache.unomi.cluster.cfg
+ </file>
+ <type>cfg</type>
+ <classifier>clustercfg</classifier>
+ </artifact>
+ <artifact>
+ <file>src/main/resources/hazelcast.xml</file>
+ <type>xml</type>
+ <classifier>hazelcastconfig</classifier>
+ </artifact>
</artifacts>
</configuration>
</execution>
[13/50] [abbrv] incubator-unomi git commit:
[jgitflow-maven-plugin]updating poms for unomi_70_es_5_x version
Posted by sh...@apache.org.
[jgitflow-maven-plugin]updating poms for unomi_70_es_5_x version
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/ff87e2f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/ff87e2f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/ff87e2f5
Branch: refs/heads/master
Commit: ff87e2f5567ad83da6b140e431bf16f31cd0b1f2
Parents: a4955ca
Author: Serge Huber <sh...@apache.org>
Authored: Mon Dec 5 09:51:42 2016 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Mon Dec 5 09:51:42 2016 +0100
----------------------------------------------------------------------
api/pom.xml | 2 +-
extensions/geonames/pom.xml | 2 +-
extensions/geonames/rest/pom.xml | 8 +++---
extensions/geonames/services/pom.xml | 8 +++---
extensions/lists-extension/pom.xml | 4 +--
extensions/lists-extension/rest/pom.xml | 8 +++---
extensions/lists-extension/services/pom.xml | 8 +++---
extensions/pom.xml | 2 +-
extensions/privacy-extension/pom.xml | 4 +--
extensions/privacy-extension/rest/pom.xml | 8 +++---
extensions/privacy-extension/services/pom.xml | 8 +++---
itests/pom.xml | 8 +++---
kar/pom.xml | 30 ++++++++++----------
package/pom.xml | 4 +--
performance-tests/pom.xml | 4 +--
persistence-elasticsearch/core/pom.xml | 6 ++--
persistence-elasticsearch/plugins/pom.xml | 2 +-
.../plugins/security/pom.xml | 2 +-
persistence-elasticsearch/pom.xml | 2 +-
persistence-spi/pom.xml | 4 +--
plugins/baseplugin/pom.xml | 6 ++--
plugins/hover-event/pom.xml | 4 +--
plugins/mail/pom.xml | 4 +--
plugins/optimization-test/pom.xml | 2 +-
plugins/past-event/pom.xml | 6 ++--
plugins/pom.xml | 4 +--
plugins/request/pom.xml | 2 +-
plugins/tracked-event/pom.xml | 6 ++--
pom.xml | 2 +-
rest/pom.xml | 6 ++--
samples/pom.xml | 2 +-
samples/tweet-button-plugin/pom.xml | 4 +--
services/pom.xml | 6 ++--
wab/pom.xml | 6 ++--
34 files changed, 92 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/api/pom.xml
----------------------------------------------------------------------
diff --git a/api/pom.xml b/api/pom.xml
index 0f06620..cdf4fd8 100644
--- a/api/pom.xml
+++ b/api/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/extensions/geonames/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/geonames/pom.xml b/extensions/geonames/pom.xml
index f601a7c..61ae8f9 100644
--- a/extensions/geonames/pom.xml
+++ b/extensions/geonames/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-extensions</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>cxs-geonames</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/extensions/geonames/rest/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/geonames/rest/pom.xml b/extensions/geonames/rest/pom.xml
index 6798bfe..72a8e74 100644
--- a/extensions/geonames/rest/pom.xml
+++ b/extensions/geonames/rest/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-geonames</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -33,14 +33,14 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-geonames-services</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
@@ -81,7 +81,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/extensions/geonames/services/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/geonames/services/pom.xml b/extensions/geonames/services/pom.xml
index e1e7310..3c1fc0b 100644
--- a/extensions/geonames/services/pom.xml
+++ b/extensions/geonames/services/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-geonames</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -28,20 +28,20 @@
<artifactId>cxs-geonames-services</artifactId>
<name>Apache Unomi :: Extensions :: Geonames Database :: Service</name>
<description>Service implementation for the Apache Unomi Context Server extension that integrates with the Geonames database</description>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<packaging>bundle</packaging>
<dependencies>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/extensions/lists-extension/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/lists-extension/pom.xml b/extensions/lists-extension/pom.xml
index d348ad9..b5bd423 100644
--- a/extensions/lists-extension/pom.xml
+++ b/extensions/lists-extension/pom.xml
@@ -27,12 +27,12 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-extensions</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>cxs-lists-extension</artifactId>
<name>Apache Unomi :: Extensions :: Lists</name>
<description>List extension for the Apache Unomi Context Server</description>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<packaging>pom</packaging>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/extensions/lists-extension/rest/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/lists-extension/rest/pom.xml b/extensions/lists-extension/rest/pom.xml
index 7a1f6b6..0988a06 100644
--- a/extensions/lists-extension/rest/pom.xml
+++ b/extensions/lists-extension/rest/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>cxs-lists-extension</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -34,14 +34,14 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-lists-extension-services</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
@@ -82,7 +82,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/extensions/lists-extension/services/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/lists-extension/services/pom.xml b/extensions/lists-extension/services/pom.xml
index d7ebedb..75f65f9 100644
--- a/extensions/lists-extension/services/pom.xml
+++ b/extensions/lists-extension/services/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>cxs-lists-extension</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -28,20 +28,20 @@
<name>Apache Unomi :: Extensions :: Lists :: Service</name>
<description>List extension service implementation for the Apache Unomi Context Server</description>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<packaging>bundle</packaging>
<dependencies>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/extensions/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/pom.xml b/extensions/pom.xml
index aa9cb48..e556570 100644
--- a/extensions/pom.xml
+++ b/extensions/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-extensions</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/extensions/privacy-extension/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/privacy-extension/pom.xml b/extensions/privacy-extension/pom.xml
index 46e4315..b23038c 100644
--- a/extensions/privacy-extension/pom.xml
+++ b/extensions/privacy-extension/pom.xml
@@ -27,12 +27,12 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-extensions</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>cxs-privacy-extension</artifactId>
<name>Apache Unomi :: Extensions :: Privacy</name>
<description>Privacy management extension for the Apache Unomi Context Server</description>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<packaging>pom</packaging>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/extensions/privacy-extension/rest/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/privacy-extension/rest/pom.xml b/extensions/privacy-extension/rest/pom.xml
index d6800cf..2bc7560 100644
--- a/extensions/privacy-extension/rest/pom.xml
+++ b/extensions/privacy-extension/rest/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>cxs-privacy-extension</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -33,14 +33,14 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-privacy-extension-services</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
@@ -81,7 +81,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/extensions/privacy-extension/services/pom.xml
----------------------------------------------------------------------
diff --git a/extensions/privacy-extension/services/pom.xml b/extensions/privacy-extension/services/pom.xml
index 4afcde8..b685e84 100644
--- a/extensions/privacy-extension/services/pom.xml
+++ b/extensions/privacy-extension/services/pom.xml
@@ -20,27 +20,27 @@
<parent>
<artifactId>cxs-privacy-extension</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>cxs-privacy-extension-services</artifactId>
<name>Apache Unomi :: Extensions :: Privacy :: Services</name>
<description>Privacy management extension service implementation for the Apache Unomi Context Server</description>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<packaging>bundle</packaging>
<dependencies>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/itests/pom.xml
----------------------------------------------------------------------
diff --git a/itests/pom.xml b/itests/pom.xml
index e0426ce..b377177 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-itests</artifactId>
<name>Apache Unomi :: Integration Tests</name>
@@ -31,19 +31,19 @@
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-kar</artifactId>
<classifier>features</classifier>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<type>xml</type>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-wab</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/kar/pom.xml
----------------------------------------------------------------------
diff --git a/kar/pom.xml b/kar/pom.xml
index 7d6c400..cee6add 100644
--- a/kar/pom.xml
+++ b/kar/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-kar</artifactId>
@@ -33,72 +33,72 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-wab</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-services</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins-base</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins-request</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins-mail</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins-optimization-test</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-lists-extension-services</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-lists-extension-rest</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-geonames-services</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-geonames-rest</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-privacy-extension-services</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-privacy-extension-rest</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.servicemix.bundles</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/package/pom.xml
----------------------------------------------------------------------
diff --git a/package/pom.xml b/package/pom.xml
index b3f7448..e2a8d1c 100644
--- a/package/pom.xml
+++ b/package/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi</artifactId>
@@ -75,7 +75,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-kar</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<classifier>features</classifier>
<type>xml</type>
<scope>runtime</scope>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/performance-tests/pom.xml
----------------------------------------------------------------------
diff --git a/performance-tests/pom.xml b/performance-tests/pom.xml
index 72c7abd..9a6b4ad 100644
--- a/performance-tests/pom.xml
+++ b/performance-tests/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-performance-tests</artifactId>
<name>Apache Unomi :: Performance Tests</name>
@@ -30,7 +30,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-wab</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/persistence-elasticsearch/core/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/pom.xml b/persistence-elasticsearch/core/pom.xml
index 891da68..e633df1 100644
--- a/persistence-elasticsearch/core/pom.xml
+++ b/persistence-elasticsearch/core/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
@@ -39,13 +39,13 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/persistence-elasticsearch/plugins/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/plugins/pom.xml b/persistence-elasticsearch/plugins/pom.xml
index 9e361c0..ccb075e 100644
--- a/persistence-elasticsearch/plugins/pom.xml
+++ b/persistence-elasticsearch/plugins/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-persistence-elasticsearch-plugins</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/persistence-elasticsearch/plugins/security/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/plugins/security/pom.xml b/persistence-elasticsearch/plugins/security/pom.xml
index b029a8c..1028e95 100644
--- a/persistence-elasticsearch/plugins/security/pom.xml
+++ b/persistence-elasticsearch/plugins/security/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-plugins</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-persistence-elasticsearch-plugins-security</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/persistence-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/pom.xml b/persistence-elasticsearch/pom.xml
index 99ef177..3cace61 100644
--- a/persistence-elasticsearch/pom.xml
+++ b/persistence-elasticsearch/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-persistence-elasticsearch</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/persistence-spi/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-spi/pom.xml b/persistence-spi/pom.xml
index 68d5ba7..50bd36f 100644
--- a/persistence-spi/pom.xml
+++ b/persistence-spi/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-persistence-spi</artifactId>
@@ -33,7 +33,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/plugins/baseplugin/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/pom.xml b/plugins/baseplugin/pom.xml
index 078cfb8..4aa9d54 100644
--- a/plugins/baseplugin/pom.xml
+++ b/plugins/baseplugin/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-base</artifactId>
@@ -55,13 +55,13 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/plugins/hover-event/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/hover-event/pom.xml b/plugins/hover-event/pom.xml
index 979aa35..980cb84 100644
--- a/plugins/hover-event/pom.xml
+++ b/plugins/hover-event/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-hover-event</artifactId>
@@ -33,7 +33,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/plugins/mail/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/mail/pom.xml b/plugins/mail/pom.xml
index 2bdcdd3..146c9c4 100644
--- a/plugins/mail/pom.xml
+++ b/plugins/mail/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-mail</artifactId>
@@ -33,7 +33,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/plugins/optimization-test/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/optimization-test/pom.xml b/plugins/optimization-test/pom.xml
index f034ed2..04926da 100644
--- a/plugins/optimization-test/pom.xml
+++ b/plugins/optimization-test/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-optimization-test</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/plugins/past-event/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/past-event/pom.xml b/plugins/past-event/pom.xml
index 7f63535..520808f 100644
--- a/plugins/past-event/pom.xml
+++ b/plugins/past-event/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-past-event</artifactId>
@@ -33,13 +33,13 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/plugins/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/pom.xml b/plugins/pom.xml
index 35d1fbd..77d9535 100644
--- a/plugins/pom.xml
+++ b/plugins/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins</artifactId>
@@ -43,7 +43,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/plugins/request/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/request/pom.xml b/plugins/request/pom.xml
index 1764ed1..08e7a5a 100644
--- a/plugins/request/pom.xml
+++ b/plugins/request/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-request</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/plugins/tracked-event/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/tracked-event/pom.xml b/plugins/tracked-event/pom.xml
index bb47811..874e723 100644
--- a/plugins/tracked-event/pom.xml
+++ b/plugins/tracked-event/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-plugins</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-plugins-tracked-event</artifactId>
@@ -33,13 +33,13 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-elasticsearch-core</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e6a8f4a..aaced29 100644
--- a/pom.xml
+++ b/pom.xml
@@ -34,7 +34,7 @@
provides a high-performance user profile and event tracking server.
</description>
<url>http://unomi.incubator.apache.org</url>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<packaging>pom</packaging>
<licenses>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/rest/pom.xml
----------------------------------------------------------------------
diff --git a/rest/pom.xml b/rest/pom.xml
index bba4a5a..f795ff1 100644
--- a/rest/pom.xml
+++ b/rest/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-rest</artifactId>
@@ -46,7 +46,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
@@ -93,7 +93,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<!--<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/samples/pom.xml
----------------------------------------------------------------------
diff --git a/samples/pom.xml b/samples/pom.xml
index 8941427..20a0403 100644
--- a/samples/pom.xml
+++ b/samples/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>unomi-root</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/samples/tweet-button-plugin/pom.xml
----------------------------------------------------------------------
diff --git a/samples/tweet-button-plugin/pom.xml b/samples/tweet-button-plugin/pom.xml
index c1725d5..8d5076c 100644
--- a/samples/tweet-button-plugin/pom.xml
+++ b/samples/tweet-button-plugin/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>samples</artifactId>
<groupId>org.apache.unomi</groupId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>tweet-button-plugin</artifactId>
@@ -33,7 +33,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/services/pom.xml
----------------------------------------------------------------------
diff --git a/services/pom.xml b/services/pom.xml
index abd3cc1..c90ebba 100644
--- a/services/pom.xml
+++ b/services/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-services</artifactId>
@@ -33,7 +33,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
@@ -52,7 +52,7 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/ff87e2f5/wab/pom.xml
----------------------------------------------------------------------
diff --git a/wab/pom.xml b/wab/pom.xml
index 5af699f..d89cceb 100644
--- a/wab/pom.xml
+++ b/wab/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-root</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
</parent>
<artifactId>unomi-wab</artifactId>
@@ -42,13 +42,13 @@
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-api</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>unomi-persistence-spi</artifactId>
- <version>1.2.0-incubating-SNAPSHOT</version>
+ <version>1.2.0-incubating-unomi_70_es_5_x-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
[31/50] [abbrv] incubator-unomi git commit: UNOMI-70 : Moved keyword
field test to elasticsearch service
Posted by sh...@apache.org.
UNOMI-70 : Moved keyword field test to elasticsearch service
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/29423931
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/29423931
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/29423931
Branch: refs/heads/master
Commit: 2942393130484159114c4824e49b815f6d45d981
Parents: ac2c9ba
Author: Thomas Draier <dr...@apache.org>
Authored: Wed Dec 21 17:05:11 2016 +0100
Committer: Thomas Draier <dr...@apache.org>
Committed: Wed Dec 21 17:05:11 2016 +0100
----------------------------------------------------------------------
.../ElasticSearchPersistenceServiceImpl.java | 39 +++++++++++---------
.../services/services/GoalsServiceImpl.java | 7 ----
.../services/services/QueryServiceImpl.java | 7 ----
3 files changed, 21 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/29423931/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index e7ed75b..6c1d8e0 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -24,10 +24,7 @@ import org.apache.karaf.cellar.core.*;
import org.apache.karaf.cellar.core.control.SwitchStatus;
import org.apache.karaf.cellar.core.event.EventProducer;
import org.apache.karaf.cellar.core.event.EventType;
-import org.apache.unomi.api.ClusterNode;
-import org.apache.unomi.api.Item;
-import org.apache.unomi.api.PartialList;
-import org.apache.unomi.api.TimestampedItem;
+import org.apache.unomi.api.*;
import org.apache.unomi.api.conditions.Condition;
import org.apache.unomi.api.query.DateRange;
import org.apache.unomi.api.query.IpRange;
@@ -1075,6 +1072,17 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
return null;
}
+ private String getPropertyNameWithData(String name, String itemType) {
+ Map<String,Object> propertyMapping = getPropertyMapping(name,itemType);
+ if (propertyMapping != null
+ && "text".equals(propertyMapping.get("type"))
+ && propertyMapping.containsKey("fields")
+ && ((Map)propertyMapping.get("fields")).containsKey("keyword")) {
+ name += ".keyword";
+ }
+ return name;
+ }
+
public boolean saveQuery(final String queryName, final String query) {
return new InClassLoaderExecute<Boolean>() {
protected Boolean execute(Object... args) {
@@ -1279,14 +1287,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
requestBuilder = requestBuilder.addSort(distanceSortBuilder.order(SortOrder.ASC));
}
} else {
- String name = StringUtils.substringBeforeLast(sortByElement,":");
- Map<String,Object> propertyMapping = getPropertyMapping(name,itemType);
- if (propertyMapping != null
- && "text".equals(propertyMapping.get("type"))
- && propertyMapping.containsKey("fields")
- && ((Map)propertyMapping.get("fields")).containsKey("keyword")) {
- name += ".keyword";
- }
+ String name = getPropertyNameWithData(StringUtils.substringBeforeLast(sortByElement,":"), itemType);
if (sortByElement.endsWith(":desc")) {
requestBuilder = requestBuilder.addSort(name, SortOrder.DESC);
} else {
@@ -1397,15 +1398,16 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
if (aggregate != null) {
AggregationBuilder bucketsAggregation = null;
+ String fieldName = aggregate.getField();
if (aggregate instanceof DateAggregate) {
DateAggregate dateAggregate = (DateAggregate) aggregate;
- DateHistogramAggregationBuilder dateHistogramBuilder = AggregationBuilders.dateHistogram("buckets").field(aggregate.getField()).dateHistogramInterval(new DateHistogramInterval((dateAggregate.getInterval())));
+ DateHistogramAggregationBuilder dateHistogramBuilder = AggregationBuilders.dateHistogram("buckets").field(fieldName).dateHistogramInterval(new DateHistogramInterval((dateAggregate.getInterval())));
if (dateAggregate.getFormat() != null) {
dateHistogramBuilder.format(dateAggregate.getFormat());
}
bucketsAggregation = dateHistogramBuilder;
} else if (aggregate instanceof NumericRangeAggregate) {
- RangeAggregationBuilder rangebuilder = AggregationBuilders.range("buckets").field(aggregate.getField());
+ RangeAggregationBuilder rangebuilder = AggregationBuilders.range("buckets").field(fieldName);
for (NumericRange range : ((NumericRangeAggregate) aggregate).getRanges()) {
if (range != null) {
if (range.getFrom() != null && range.getTo() != null) {
@@ -1420,7 +1422,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
bucketsAggregation = rangebuilder;
} else if (aggregate instanceof DateRangeAggregate) {
DateRangeAggregate dateRangeAggregate = (DateRangeAggregate) aggregate;
- DateRangeAggregationBuilder rangebuilder = AggregationBuilders.dateRange("buckets").field(aggregate.getField());
+ DateRangeAggregationBuilder rangebuilder = AggregationBuilders.dateRange("buckets").field(fieldName);
if (dateRangeAggregate.getFormat() != null) {
rangebuilder.format(dateRangeAggregate.getFormat());
}
@@ -1432,7 +1434,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
bucketsAggregation = rangebuilder;
} else if (aggregate instanceof IpRangeAggregate) {
IpRangeAggregate ipRangeAggregate = (IpRangeAggregate) aggregate;
- IpRangeAggregationBuilder rangebuilder = AggregationBuilders.ipRange("buckets").field(aggregate.getField());
+ IpRangeAggregationBuilder rangebuilder = AggregationBuilders.ipRange("buckets").field(fieldName);
for (IpRange range : ipRangeAggregate.getRanges()) {
if (range != null) {
rangebuilder.addRange(range.getKey(), range.getFrom(), range.getTo());
@@ -1440,11 +1442,12 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
bucketsAggregation = rangebuilder;
} else {
+ fieldName = getPropertyNameWithData(fieldName, itemType);
//default
- bucketsAggregation = AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);
+ bucketsAggregation = AggregationBuilders.terms("buckets").field(fieldName).size(5000);
}
if (bucketsAggregation != null) {
- final MissingAggregationBuilder missingBucketsAggregation = AggregationBuilders.missing("missing").field(aggregate.getField());
+ final MissingAggregationBuilder missingBucketsAggregation = AggregationBuilders.missing("missing").field(fieldName);
for (AggregationBuilder aggregationBuilder : lastAggregation) {
bucketsAggregation.subAggregation(aggregationBuilder);
missingBucketsAggregation.subAggregation(aggregationBuilder);
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/29423931/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java
----------------------------------------------------------------------
diff --git a/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java
index a6bd1d6..a5af23f 100644
--- a/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java
@@ -493,13 +493,6 @@ public class GoalsServiceImpl implements GoalsService, SynchronousBundleListener
}
if(aggregate == null){
- Map<String,Object> propertyMapping = persistenceService.getPropertyMapping(property, Session.ITEM_TYPE);
- if (propertyMapping != null
- && "text".equals(propertyMapping.get("type"))
- && propertyMapping.containsKey("fields")
- && ((Map)propertyMapping.get("fields")).containsKey("keyword")) {
- property += ".keyword";
- }
aggregate = new TermsAggregate(property);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/29423931/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
----------------------------------------------------------------------
diff --git a/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
index 30e6dab..33e4f47 100644
--- a/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
@@ -83,13 +83,6 @@ public class QueryServiceImpl implements QueryService {
}
// fall back on terms aggregate
- Map<String,Object> propertyMapping = persistenceService.getPropertyMapping( property, itemType);
- if (propertyMapping != null
- && "text".equals(propertyMapping.get("type"))
- && propertyMapping.containsKey("fields")
- && ((Map)propertyMapping.get("fields")).containsKey("keyword")) {
- property += ".keyword";
- }
return persistenceService.aggregateQuery(query.getCondition(), new TermsAggregate(property), itemType);
}