You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by sh...@apache.org on 2016/08/09 13:55:28 UTC
[1/2] incubator-unomi git commit: UNOMI-28 Upgrade to ElasticSearch
2.x
Repository: incubator-unomi
Updated Branches:
refs/heads/UNOMI-28-ES-2-X-UPGRADE [created] 131a06aa7
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/resources/elasticsearch.yml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/elasticsearch.yml b/persistence-elasticsearch/core/src/main/resources/elasticsearch.yml
index 293f734..fe0e52f 100644
--- a/persistence-elasticsearch/core/src/main/resources/elasticsearch.yml
+++ b/persistence-elasticsearch/core/src/main/resources/elasticsearch.yml
@@ -1,269 +1,103 @@
-##################### ElasticSearch Configuration Example #####################
-
-# This file contains an overview of various configuration settings,
-# targeted at operations staff. Application developers should
-# consult the guide at <http://elasticsearch.org/guide>.
+# ======================== Elasticsearch Configuration =========================
#
-# The installation procedure is covered at
-# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/setup.html>.
+# NOTE: Elasticsearch comes with reasonable defaults for most settings.
+# Before you set out to tweak and tune the configuration, make sure you
+# understand what are you trying to accomplish and the consequences.
#
-# ElasticSearch comes with reasonable defaults for most settings,
-# so you can try it out without bothering with configuration.
+# The primary way of configuring a node is via this file. This template lists
+# the most important settings you may want to configure for a production cluster.
#
-# Most of the time, these defaults are just fine for running a production
-# cluster. If you're fine-tuning your cluster, or wondering about the
-# effect of certain configuration option, please _do ask_ on the
-# mailing list or IRC channel [http://elasticsearch.org/community].
-
-# Any element in the configuration can be replaced with environment variables
-# by placing them in ${...} notation. For example:
+# Please see the documentation for further information on configuration options:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration.html>
#
-# node.rack: ${RACK_ENV_VAR}
-
-# For information on supported formats and syntax for the config file, see
-# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/setup-configuration.html>
-
-
-################################### Cluster ###################################
-
-# Cluster name identifies your cluster for auto-discovery. If you're running
-# multiple clusters on the same network, make sure you're using unique names.
+# ---------------------------------- Cluster -----------------------------------
#
-# cluster.name: contextElasticSearch
-
-
-#################################### Node #####################################
-
-# Node names are generated dynamically on startup, so you're relieved
-# from configuring them manually. You can tie this node to a specific name:
-#
-# node.name: "od-fts1a"
-
-# Every node can be configured to allow or deny being eligible as the master,
-# and to allow or deny to store the data.
+# Use a descriptive name for your cluster:
#
-# Allow this node to be eligible as a master node (enabled by default):
+# cluster.name: my-application
#
-# node.master: true
+# ------------------------------------ Node ------------------------------------
#
-# Allow this node to store data (enabled by default):
+# Use a descriptive name for the node:
#
-# node.data: true
-
-# You can exploit these settings to design advanced cluster topologies.
+# node.name: node-1
#
-# 1. You want this node to never become a master node, only to hold data.
-# This will be the "workhorse" of your cluster.
+# Add custom attributes to the node:
#
-# node.master: false
-# node.data: true
+# node.rack: r1
#
-# 2. You want this node to only serve as a master: to not store any data and
-# to have free resources. This will be the "coordinator" of your cluster.
+# ----------------------------------- Paths ------------------------------------
#
-# node.master: true
-# node.data: false
+# Path to directory where to store the data (separate multiple locations by comma):
#
-# 3. You want this node to be neither master nor data node, but
-# to act as a "search load balancer" (fetching data from nodes,
-# aggregating results, etc.)
-#
-# node.master: false
-# node.data: false
-
-# Use the Cluster Health API [http://localhost:9200/_cluster/health], the
-# Node Info API [http://localhost:9200/_cluster/nodes] or GUI tools
-# such as <http://github.com/lukas-vlcek/bigdesk> and
-# <http://mobz.github.com/elasticsearch-head> to inspect the cluster state.
-
-# A node can have generic attributes associated with it, which can later be used
-# for customized shard allocation filtering, or allocation awareness. An attribute
-# is a simple key value pair, similar to node.key: value, here is an example:
+# path.data: /path/to/data
#
-# node.rack: rack314
-
-# By default, multiple nodes are allowed to start from the same installation location
-# to disable it, set the following:
-# node.max_local_storage_nodes: 1
-
-
-#################################### Index ####################################
-
-# You can set a number of options (such as shard/replica options, mapping
-# or analyzer definitions, translog settings, ...) for indices globally,
-# in this file.
+# Path to log files:
#
-# Note, that it makes more sense to configure index settings specifically for
-# a certain index, either when creating it or by using the index templates API.
+# path.logs: /path/to/logs
#
-# See <http://elasticsearch.org/guide/en/elasticsearch/reference/current/index-modules.html> and
-# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/indices-create-index.html>
-# for more information.
-
-# Set the number of shards (splits) of an index (5 by default):
+# ----------------------------------- Memory -----------------------------------
#
-# index.number_of_shards: 2
-
-# Set the number of replicas (additional copies) of an index (1 by default):
+# Lock the memory on startup:
#
-# index.number_of_replicas: 1
-
-# Note, that for development on a local machine, with small indices, it usually
-# makes sense to "disable" the distributed features:
+# bootstrap.mlockall: true
#
-# index.number_of_shards: 1
-# index.number_of_replicas: 0
-
-# These settings directly affect the performance of index and search operations
-# in your cluster. Assuming you have enough machines to hold shards and
-# replicas, the rule of thumb is:
+# Make sure that the `ES_HEAP_SIZE` environment variable is set to about half the memory
+# available on the system and that the owner of the process is allowed to use this limit.
#
-# 1. Having more *shards* enhances the _indexing_ performance and allows to
-# _distribute_ a big index across machines.
-# 2. Having more *replicas* enhances the _search_ performance and improves the
-# cluster _availability_.
+# Elasticsearch performs poorly when the system is swapping the memory.
#
-# The "number_of_shards" is a one-time setting for an index.
+# ---------------------------------- Network -----------------------------------
#
-# The "number_of_replicas" can be increased or decreased anytime,
-# by using the Index Update Settings API.
+# Set the bind address to a specific IP (IPv4 or IPv6):
#
-# ElasticSearch takes care about load balancing, relocating, gathering the
-# results from nodes, etc. Experiment with different settings to fine-tune
-# your setup.
-
-# Use the Index Status API (<http://localhost:9200/A/_status>) to inspect
-# the index status.
-
-http.cors.enabled: true
-index.percolator.allow_unmapped_fields: true
-
-#################################### Paths ####################################
-
-# Path to directory containing configuration (this file and logging.yml):
+# network.host: 192.168.0.1
#
-# path.conf: /path/to/conf
-
-# Path to directory where to store index data allocated for this node.
+# Set a custom port for HTTP:
#
-# path.data: /path/to/data
+# http.port: 9200
#
-# Can optionally include more than one location, causing data to be striped across
-# the locations (a la RAID 0) on a file level, favouring locations with most free
-# space on creation. For example:
+# For more information, see the documentation at:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html>
#
-# path.data: /path/to/data1,/path/to/data2
-
-# Path to temporary files:
+# --------------------------------- Discovery ----------------------------------
#
-# path.work: /path/to/work
-
-# Path to log files:
+# Pass an initial list of hosts to perform discovery when new node is started:
+# The default list of hosts is ["127.0.0.1", "[::1]"]
#
-# path.logs: /path/to/logs
-
-# Path to where plugins are installed:
+# discovery.zen.ping.unicast.hosts: ["host1", "host2"]
#
-# path.plugins: /path/to/plugins
-
-
-#################################### Plugin ###################################
-
-# If a plugin listed here is not installed for current node, the node will not start.
+# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
#
-# plugin.mandatory: mapper-attachments,lang-groovy
-
-
-################################### Memory ####################################
-
-# ElasticSearch performs poorly when JVM starts swapping: you should ensure that
-# it _never_ swaps.
+# discovery.zen.minimum_master_nodes: 3
#
-# Set this property to true to lock the memory:
+# For more information, see the documentation at:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html>
#
-# bootstrap.mlockall: true
-
-# Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set
-# to the same value, and that the machine has enough memory to allocate
-# for ElasticSearch, leaving enough memory for the operating system itself.
+# ---------------------------------- Gateway -----------------------------------
#
-# You should also make sure that the ElasticSearch process is allowed to lock
-# the memory, eg. by using `ulimit -l unlimited`.
-
-
-############################## Network And HTTP ###############################
-
-# ElasticSearch, by default, binds itself to the 0.0.0.0 address, and listens
-# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node
-# communication. (the range means that if the port is busy, it will automatically
-# try the next port).
-
-# Set the bind address specifically (IPv4 or IPv6):
+# Block initial recovery after a full cluster restart until N nodes are started:
#
-# network.bind_host: 192.168.0.1
-
-# Set the address other nodes will use to communicate with this node. If not
-# set, it is automatically derived. It must point to an actual IP address.
+# gateway.recover_after_nodes: 3
#
-# network.publish_host: 192.168.0.1
-
-# Set both 'bind_host' and 'publish_host':
+# For more information, see the documentation at:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-gateway.html>
#
-# network.host: 192.168.0.1
-
-# Set a custom port for the node to node communication (9300 by default):
+# ---------------------------------- Various -----------------------------------
#
-# transport.tcp.port: 9300
-
-# Enable compression for all communication between nodes (disabled by default):
+# Disable starting multiple nodes on a single system:
#
-# transport.tcp.compress: true
-
-# Set a custom port to listen for HTTP traffic:
-#
-# http.port: 9200
-
-# Set a custom allowed content length:
-#
-# http.max_content_length: 100mb
-
-# Disable HTTP completely:
-#
-# http.enabled: false
-
-
-################################### Gateway ###################################
-
-# The gateway allows for persisting the cluster state between full cluster
-# restarts. Every change to the state (such as adding an index) will be stored
-# in the gateway, and when the cluster starts up for the first time,
-# it will read its state from the gateway.
-
-# There are several types of gateway implementations. For more information, see
-# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/modules-gateway.html>.
-
-# The default gateway type is the "local" gateway (recommended):
+# node.max_local_storage_nodes: 1
#
-# gateway.type: local
-
-# Settings below control how and when to start the initial recovery process on
-# a full cluster restart (to reuse as much local data as possible when using shared
-# gateway).
-
-# Allow recovery process after N nodes in a cluster are up:
+# Require explicit names when deleting indices:
#
-# gateway.recover_after_nodes: 1
-
-# Set the timeout to initiate the recovery process, once the N nodes
-# from previous setting are up (accepts time value):
+# action.destructive_requires_name: true
#
-# gateway.recover_after_time: 10m
-
-# Set how many nodes are expected in this cluster. Once these N nodes
-# are up (and recover_after_nodes is met), begin recovery process immediately
-# (without waiting for recover_after_time to expire):
+# --------------------------- Apache Unomi specific ---------------------------
#
-# gateway.expected_nodes: 2
+threadpool.index.queue_size: 1000
+script.engine.groovy.inline.update: on
+index.percolator.allow_unmapped_fields: true
# Require explicit index creation
action.auto_create_index: false
@@ -274,106 +108,3 @@ action.auto_create_index: false
#action.disable_close_all_indices: true
#action.disable_delete_all_indices: true
#action.disable_shutdown: true
-
-############################# Recovery Throttling #############################
-
-# These settings allow to control the process of shards allocation between
-# nodes during initial recovery, replica allocation, rebalancing,
-# or when adding and removing nodes.
-
-# Set the number of concurrent recoveries happening on a node:
-#
-# 1. During the initial recovery
-#
-# cluster.routing.allocation.node_initial_primaries_recoveries: 4
-#
-# 2. During adding/removing nodes, rebalancing, etc
-#
-# cluster.routing.allocation.node_concurrent_recoveries: 2
-
-# Set to throttle throughput when recovering (eg. 100mb, by default 20mb):
-#
-#indices.recovery.max_bytes_per_sec: 100mb
-
-# Set to limit the number of open concurrent streams when
-# recovering a shard from a peer:
-#
-# indices.recovery.concurrent_streams: 5
-
-
-################################## Discovery ##################################
-
-# Discovery infrastructure ensures nodes can be found within a cluster
-# and master node is elected. Multicast discovery is the default.
-
-# Set to ensure a node sees N other master eligible nodes to be considered
-# operational within the cluster. Its recommended to set it to a higher value
-# than 1 when running more than 2 nodes in the cluster.
-#
-# discovery.zen.minimum_master_nodes: 1
-
-# Set the time to wait for ping responses from other nodes when discovering.
-# Set this option to a higher value on a slow or congested network
-# to minimize discovery failures:
-#
-# discovery.zen.ping.timeout: 3s
-
-# For more information, see
-# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/modules-discovery-zen.html>
-
-# Unicast discovery allows to explicitly control which nodes will be used
-# to discover the cluster. It can be used when multicast is not present,
-# or to restrict the cluster communication-wise.
-#
-# 1. Disable multicast discovery (enabled by default):
-#
-# discovery.zen.ping.multicast.enabled: false
-#
-# 2. Configure an initial list of master nodes in the cluster
-# to perform discovery when new nodes (master or data) are started:
-#
-# discovery.zen.ping.unicast.hosts: ["host1", "host2:port"]
-
-# EC2 discovery allows to use AWS EC2 API in order to perform discovery.
-#
-# You have to install the cloud-aws plugin for enabling the EC2 discovery.
-#
-# For more information, see
-# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/modules-discovery-ec2.html>
-#
-# See <http://elasticsearch.org/tutorials/elasticsearch-on-ec2/>
-# for a step-by-step tutorial.
-
-
-################################## Slow Log ##################################
-
-# Shard level query and fetch threshold logging.
-
-#index.search.slowlog.threshold.query.warn: 10s
-#index.search.slowlog.threshold.query.info: 5s
-#index.search.slowlog.threshold.query.debug: 2s
-#index.search.slowlog.threshold.query.trace: 500ms
-
-#index.search.slowlog.threshold.fetch.warn: 1s
-#index.search.slowlog.threshold.fetch.info: 800ms
-#index.search.slowlog.threshold.fetch.debug: 500ms
-#index.search.slowlog.threshold.fetch.trace: 200ms
-
-#index.indexing.slowlog.threshold.index.warn: 10s
-#index.indexing.slowlog.threshold.index.info: 5s
-#index.indexing.slowlog.threshold.index.debug: 2s
-#index.indexing.slowlog.threshold.index.trace: 500ms
-
-################################## GC Logging ################################
-
-#monitor.jvm.gc.ParNew.warn: 1000ms
-#monitor.jvm.gc.ParNew.info: 700ms
-#monitor.jvm.gc.ParNew.debug: 400ms
-
-#monitor.jvm.gc.ConcurrentMarkSweep.warn: 10s
-#monitor.jvm.gc.ConcurrentMarkSweep.info: 5s
-#monitor.jvm.gc.ConcurrentMarkSweep.debug: 2s
-
-################################## Thread Pool ###############################
-threadpool.index.queue_size: 1000
-script.engine.groovy.inline.update: on
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscapeTest.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscapeTest.java b/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscapeTest.java
new file mode 100644
index 0000000..c4322dc
--- /dev/null
+++ b/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscapeTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.persistence.elasticsearch;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.*;
+
+/**
+ * A unit test class for the FieldDotEscaper utility methods
+ */
+public class FieldDotEscapeTest {
+
+ @Test
+ public void testTrickyJson() {
+ String result = FieldDotEscaper.escapeJson("{\"tricking_the_parser\" : \"this.should.not\\\": match.either\"}");
+ Assert.assertTrue("Found escaped pattern instead of untouched one", result.contains("this.should.not\\\": match.either"));
+ }
+
+ @Test
+ public void testJustDot() {
+ String result = FieldDotEscaper.escapeJson("{\".\" : \"this.should.not\\\": match.either\"}");
+ Assert.assertTrue("Didn't find expected escaped pattern", result.contains("\"__DOT__\""));
+ Assert.assertTrue("Found escaped pattern instead of untouched one", result.contains("this.should.not\\\": match.either"));
+ }
+
+ @Test
+ public void testComplexJson() throws IOException {
+ InputStream complexJsonInputStream = this.getClass().getClassLoader().getResourceAsStream("complex.json");
+ String input = IOUtils.toString(complexJsonInputStream);
+ System.out.println("Original JSON:");
+ System.out.println("=================");
+ System.out.println(input);
+ Set<String> modifiedNames = new LinkedHashSet<>();
+ String result = FieldDotEscaper.escapeJson(input, modifiedNames);
+ System.out.println("Modified names:");
+ System.out.println("=================");
+ for (String modifiedName : modifiedNames) {
+ System.out.println(modifiedName);
+ }
+ System.out.println("Transformed JSON:");
+ System.out.println("=================");
+ System.out.println(result);
+ Assert.assertTrue("Didn't find expected escaped pattern", result.contains("src_terms[0]__DOT__fields__DOT__siteContent"));
+ Assert.assertTrue("Didn't find expected escaped pattern", result.contains("newline__DOT__test"));
+ Assert.assertTrue("Found escaped pattern instead of untouched one", result.contains("this.should.never:match"));
+ Assert.assertTrue("Found escaped pattern instead of untouched one", result.contains("this.should.not\\\": match.either"));
+ result = FieldDotEscaper.unescapeJson(result);
+ Assert.assertEquals("Round trip of escaping then unescaping should be identical", input, result);
+ }
+
+ @Test
+ public void testString() {
+ String input = "this.is.a..test";
+ String result = FieldDotEscaper.unescapeString(FieldDotEscaper.escapeString(input));
+ Assert.assertEquals("Strings should be exactly the same", input, result);
+ }
+
+ @Test
+ public void testMap() {
+ Map<String,Object> input = new HashMap<>();
+ input.put("this.is.a..test", "1");
+ input.put("another.test", "2");
+ Map<? extends String,?> result = FieldDotEscaper.unescapeMap(FieldDotEscaper.escapeMap(input));
+ Assert.assertEquals("Maps should be identical", input, result);
+ }
+
+ @Test
+ public void testProperties() {
+ Properties input = new Properties();
+ input.put("this.is.a..test", "1");
+ input.put("another.test", "2");
+ Properties result = FieldDotEscaper.unescapeProperties(FieldDotEscaper.escapeProperties(input));
+ Assert.assertEquals("Properties should be identical", input, result);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformerTest.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformerTest.java b/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformerTest.java
new file mode 100644
index 0000000..284ac84
--- /dev/null
+++ b/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformerTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.persistence.elasticsearch;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.io.IOUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
+
+/**
+ * A unit test class for our custom JSON parser.
+ */
+public class FieldDotJsonTransformerTest {
+
+ @Test
+ public void testNoReplacement() throws IOException {
+ InputStream complexJsonInputStream = this.getClass().getClassLoader().getResourceAsStream("complex.json");
+ String input = IOUtils.toString(complexJsonInputStream);
+ StringBuffer output = new StringBuffer();
+ FieldDotJsonTransformer fieldDotJsonTransformer = new FieldDotJsonTransformer(input, output, null);
+ fieldDotJsonTransformer.transform();
+ Assert.assertEquals("Parsed JSON and original should be identical", input, output.toString());
+ InputStream profileJsonInputStream = this.getClass().getClassLoader().getResourceAsStream("profile.json");
+ input = IOUtils.toString(profileJsonInputStream);
+ output = new StringBuffer();
+ fieldDotJsonTransformer = new FieldDotJsonTransformer(input, output, null);
+ fieldDotJsonTransformer.transform();
+ Assert.assertEquals("Parsed JSON and original should be identical", input, output.toString());
+ }
+
+ @Test
+ public void testWithJackson() throws IOException {
+ InputStream complexJsonInputStream = this.getClass().getClassLoader().getResourceAsStream("complex.json");
+ String input = IOUtils.toString(complexJsonInputStream);
+ StringBuffer output = new StringBuffer();
+ FieldDotJsonTransformer fieldDotJsonTransformer = new FieldDotJsonTransformer(input, output, FieldDotEscaper.DOT_ESCAPE_MARKER);
+ fieldDotJsonTransformer.transform();
+ ObjectMapper objectMapper = new ObjectMapper();
+ Map<String,Object> object = objectMapper.readValue(output.toString(), Map.class);
+ Assert.assertNotNull("Object parsed with Jackson should not be null", object);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/test/resources/complex.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/test/resources/complex.json b/persistence-elasticsearch/core/src/test/resources/complex.json
new file mode 100644
index 0000000..c6ff41c
--- /dev/null
+++ b/persistence-elasticsearch/core/src/test/resources/complex.json
@@ -0,0 +1,50 @@
+{
+ "events": [
+ {
+ "scope": "ACMESPACE",
+ "eventType": "form",
+ "source": {
+ "itemType": "page",
+ "scope": "ACMESPACE",
+ "itemId": "${pageId}",
+ "properties": {
+ "pageInfo": {
+ "pageID": "${pageId}",
+ "pageName": "${pageName}",
+ "pagePath": "${pagePath}",
+ "destinationURL": "${destinationURL}",
+ "referringURL": "${previousURL}",
+ "language": "${language}"
+ },
+ "category": {},
+ "attributes": {
+ "colon_in_value" : "this.should.never:match",
+ "newline.test"
+ : "more.dots",
+ "tricking_the_parser" : "this.should.not\": match.either",
+ "unicode black star": "before\u2605after",
+ "escaping test" : "\"\n\r\t\\"
+ }
+ }
+ },
+ "target": {
+ "itemType": "form",
+ "scope": "ACMESPACE",
+ "itemId": "searchForm"
+ },
+ "properties": {
+ "jcrMethodToCall": "get",
+ "src_originSiteKey": "ACMESPACE",
+ "src_terms[0].term": "${word}",
+ "src_terms[0].applyFilter": "true",
+ "src_terms[0].match": "all_words",
+ "src_terms[0].fields.siteContent": "true",
+ "src_terms[0].fields.tags": "true",
+ "src_terms[0].fields.files": "true",
+ "src_sites.values": "ACMESPACE",
+ "src_sitesForReferences.values": "systemsite",
+ "src_languages.values": "en"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/test/resources/profile.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/test/resources/profile.json b/persistence-elasticsearch/core/src/test/resources/profile.json
new file mode 100644
index 0000000..8fc9d2c
--- /dev/null
+++ b/persistence-elasticsearch/core/src/test/resources/profile.json
@@ -0,0 +1 @@
+{"itemId":"6bb362d8-d094-44b4-ba2e-012163286825","itemType":"personaSession","scope":null,"profileId":"googleBot","profile":{"itemId":"googleBot","itemType":"persona","properties":{"description":"Represents a Google bot","firstName":"Google","lastName":"Bot"},"systemProperties":{},"segments":[],"scores":null,"mergedWith":null},"properties":{"operatingSystemName":"unknown","sessionCountryName":"United States","location":{"lon":-122.084058,"lat":37.422},"userAgentVersion":"2.1","sessionCountryCode":"US","deviceCategory":"Other","operatingSystemFamily":"unknown","userAgentName":"Googlebot/2.1","sessionCity":"Mountain View","remoteHost":"www.google.com","remoteAddr":"66.249.66.1"},"systemProperties":{},"timeStamp":"2014-09-18T09:18:52Z","lastEventDate":"2014-09-18T11:23:59Z","size":0,"duration":7507068}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPlugin.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPlugin.java b/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPlugin.java
index 61814a1..79fc7d3 100644
--- a/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPlugin.java
+++ b/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPlugin.java
@@ -20,13 +20,13 @@ package org.apache.unomi.elasticsearch.plugin.security;
import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugins.AbstractPlugin;
+import org.elasticsearch.plugins.Plugin;
import java.util.Collection;
-import static org.elasticsearch.common.collect.Lists.newArrayList;
+import static com.google.common.collect.Lists.newArrayList;
-public class SecurityPlugin extends AbstractPlugin {
+public class SecurityPlugin extends Plugin {
public SecurityPlugin(Settings settings) {
super();
@@ -41,17 +41,17 @@ public class SecurityPlugin extends AbstractPlugin {
}
@Override
- public Collection<Class<? extends Module>> modules() {
- Collection<Class<? extends Module>> modules = newArrayList();
+ public Collection<Module> nodeModules() {
+ Collection<Module> modules = newArrayList();
// if (settings.getAsBoolean("security.enabled", true)) {
- modules.add(SecurityPluginModule.class);
+ modules.add(new SecurityPluginModule());
// }
return modules;
}
@SuppressWarnings("rawtypes")
@Override
- public Collection<Class<? extends LifecycleComponent>> services() {
+ public Collection<Class<? extends LifecycleComponent>> nodeServices() {
Collection<Class<? extends LifecycleComponent>> services = newArrayList();
// if (settings.getAsBoolean("security.enabled", true)) {
services.add(SecurityPluginService.class);
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPluginModule.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPluginModule.java b/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPluginModule.java
index 67e1770..353c05f 100644
--- a/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPluginModule.java
+++ b/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPluginModule.java
@@ -18,11 +18,10 @@
package org.apache.unomi.elasticsearch.plugin.security;
import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.settings.Settings;
public class SecurityPluginModule extends AbstractModule {
- public SecurityPluginModule(Settings settings) {
+ public SecurityPluginModule() {
super();
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPluginService.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPluginService.java b/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPluginService.java
index f3941ea..6bd73eb 100644
--- a/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPluginService.java
+++ b/persistence-elasticsearch/plugins/security/src/main/java/org/apache/unomi/elasticsearch/plugin/security/SecurityPluginService.java
@@ -56,10 +56,10 @@ public class SecurityPluginService extends AbstractLifecycleComponent<SecurityPl
super(settings);
this.restController = restController;
this.transportService = transportService;
- this.publishHost = componentSettings.get("publish_host", settings.get("transport.publish_host", settings.get("transport.host")));
+ this.publishHost = settings.get("publish_host", settings.get("transport.publish_host", settings.get("transport.host")));
InetAddress publishHostAddress = null;
try {
- publishHostAddress = networkService.resolvePublishHostAddress(publishHost);
+ publishHostAddress = networkService.resolvePublishHostAddresses(new String[] { publishHost });
} catch (IOException e) {
logger.error("Error trying to resolve publish host address " + publishHost);
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java
index c41309b..f5330ed 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java
@@ -20,10 +20,10 @@ package org.apache.unomi.plugins.baseplugin.conditions;
import org.apache.unomi.api.conditions.Condition;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -33,8 +33,8 @@ import java.util.Map;
public class BooleanConditionESQueryBuilder implements ConditionESQueryBuilder {
@Override
- public FilterBuilder buildFilter(Condition condition, Map<String, Object> context,
- ConditionESQueryBuilderDispatcher dispatcher) {
+ public QueryBuilder buildQuery(Condition condition, Map<String, Object> context,
+ ConditionESQueryBuilderDispatcher dispatcher) {
boolean isAndOperator = "and".equalsIgnoreCase((String) condition.getParameter("operator"));
@SuppressWarnings("unchecked")
List<Condition> conditions = (List<Condition>) condition.getParameter("subConditions");
@@ -45,11 +45,15 @@ public class BooleanConditionESQueryBuilder implements ConditionESQueryBuilder {
return dispatcher.buildFilter(conditions.get(0), context);
}
- FilterBuilder[] l = new FilterBuilder[conditionCount];
+ BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
for (int i = 0; i < conditionCount; i++) {
- l[i] = dispatcher.buildFilter(conditions.get(i));
+ if (isAndOperator) {
+ boolQueryBuilder.must(dispatcher.buildFilter(conditions.get(i)));
+ } else {
+ boolQueryBuilder.should(dispatcher.buildFilter(conditions.get(i)));
+ }
}
- return isAndOperator ? FilterBuilders.andFilter(l) : FilterBuilders.orFilter(l);
+ return boolQueryBuilder;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java
index 804f4f4..083cfe4 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java
@@ -20,14 +20,14 @@ package org.apache.unomi.plugins.baseplugin.conditions;
import org.apache.unomi.api.conditions.Condition;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
import java.util.Map;
public class GeoLocationByPointSessionConditionESQueryBuilder implements ConditionESQueryBuilder {
@Override
- public FilterBuilder buildFilter(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher) {
+ public QueryBuilder buildQuery(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher) {
String type = (String) condition.getParameter("type");
if("circle".equals(type)) {
@@ -36,7 +36,7 @@ public class GeoLocationByPointSessionConditionESQueryBuilder implements Conditi
String distance = condition.getParameter("distance").toString();
if(circleLatitude != null && circleLongitude != null && distance != null) {
- return FilterBuilders.geoDistanceFilter("location")
+ return QueryBuilders.geoDistanceQuery("location")
.lat(circleLatitude)
.lon(circleLongitude)
.distance(distance);
@@ -48,7 +48,7 @@ public class GeoLocationByPointSessionConditionESQueryBuilder implements Conditi
Double rectLongitudeSW = (Double) condition.getParameter("rectLongitudeSW");
if(rectLatitudeNE != null && rectLongitudeNE != null && rectLatitudeSW != null && rectLongitudeSW != null) {
- return FilterBuilders.geoBoundingBoxFilter("location")
+ return QueryBuilders.geoBoundingBoxQuery("location")
.topLeft(rectLatitudeNE, rectLongitudeNE)
.bottomRight(rectLatitudeSW, rectLongitudeSW);
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/MatchAllConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/MatchAllConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/MatchAllConditionESQueryBuilder.java
index 9efd2c1..700286a 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/MatchAllConditionESQueryBuilder.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/MatchAllConditionESQueryBuilder.java
@@ -20,15 +20,15 @@ package org.apache.unomi.plugins.baseplugin.conditions;
import org.apache.unomi.api.conditions.Condition;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
import java.util.Map;
public class MatchAllConditionESQueryBuilder implements ConditionESQueryBuilder {
@Override
- public FilterBuilder buildFilter(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher) {
- return FilterBuilders.matchAllFilter();
+ public QueryBuilder buildQuery(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher) {
+ return QueryBuilders.matchAllQuery();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NotConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NotConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NotConditionESQueryBuilder.java
index 7793aa0..9574fe8 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NotConditionESQueryBuilder.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NotConditionESQueryBuilder.java
@@ -20,8 +20,8 @@ package org.apache.unomi.plugins.baseplugin.conditions;
import org.apache.unomi.api.conditions.Condition;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
import java.util.Map;
@@ -30,8 +30,8 @@ import java.util.Map;
*/
public class NotConditionESQueryBuilder implements ConditionESQueryBuilder {
- public FilterBuilder buildFilter(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher) {
+ public QueryBuilder buildQuery(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher) {
Condition subCondition = (Condition) condition.getParameter("subCondition");
- return FilterBuilders.notFilter(dispatcher.buildFilter(subCondition, context));
+ return QueryBuilders.boolQuery().mustNot(dispatcher.buildFilter(subCondition, context));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionESQueryBuilder.java
index 1472cf9..84d964f 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionESQueryBuilder.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionESQueryBuilder.java
@@ -26,8 +26,8 @@ import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBui
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher;
import org.apache.unomi.persistence.spi.PersistenceService;
import org.apache.unomi.persistence.spi.aggregate.TermsAggregate;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
import java.util.*;
@@ -43,7 +43,7 @@ public class PastEventConditionESQueryBuilder implements ConditionESQueryBuilder
this.persistenceService = persistenceService;
}
- public FilterBuilder buildFilter(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher) {
+ public QueryBuilder buildQuery(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher) {
Condition eventCondition;
try {
eventCondition = (Condition) condition.getParameter("eventCondition");
@@ -84,6 +84,6 @@ public class PastEventConditionESQueryBuilder implements ConditionESQueryBuilder
}
}
- return FilterBuilders.idsFilter(Profile.ITEM_TYPE).addIds(ids.toArray(new String[ids.size()]));
+ return QueryBuilders.idsQuery(Profile.ITEM_TYPE).addIds(ids.toArray(new String[ids.size()]));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
index d063ff3..23f1df8 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java
@@ -22,9 +22,10 @@ import org.apache.unomi.api.conditions.Condition;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionContextHelper;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher;
-import org.elasticsearch.common.joda.time.DateTime;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.joda.time.DateTime;
import java.util.List;
import java.util.Map;
@@ -35,7 +36,7 @@ public class PropertyConditionESQueryBuilder implements ConditionESQueryBuilder
}
@Override
- public FilterBuilder buildFilter(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher) {
+ public QueryBuilder buildQuery(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher) {
String op = (String) condition.getParameter("comparisonOperator");
String name = (String) condition.getParameter("propertyName");
@@ -60,56 +61,60 @@ public class PropertyConditionESQueryBuilder implements ConditionESQueryBuilder
switch (op) {
case "equals":
checkRequiredValue(value, name, op, false);
- return FilterBuilders.termFilter(name, value);
+ return QueryBuilders.termQuery(name, value);
case "notEquals":
checkRequiredValue(value, name, op, false);
- return FilterBuilders.notFilter(FilterBuilders.termFilter(name, value));
+ return QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(name, value));
case "greaterThan":
checkRequiredValue(value, name, op, false);
- return FilterBuilders.rangeFilter(name).gt(value);
+ return QueryBuilders.rangeQuery(name).gt(value);
case "greaterThanOrEqualTo":
checkRequiredValue(value, name, op, false);
- return FilterBuilders.rangeFilter(name).gte(value);
+ return QueryBuilders.rangeQuery(name).gte(value);
case "lessThan":
checkRequiredValue(value, name, op, false);
- return FilterBuilders.rangeFilter(name).lt(value);
+ return QueryBuilders.rangeQuery(name).lt(value);
case "lessThanOrEqualTo":
checkRequiredValue(value, name, op, false);
- return FilterBuilders.rangeFilter(name).lte(value);
+ return QueryBuilders.rangeQuery(name).lte(value);
case "between":
checkRequiredValuesSize(values, name, op, 2);
- return FilterBuilders.rangeFilter(name).gte(values.get(0)).lte(values.get(1));
+ return QueryBuilders.rangeQuery(name).gte(values.get(0)).lte(values.get(1));
case "exists":
- return FilterBuilders.existsFilter(name);
+ return QueryBuilders.existsQuery(name);
case "missing":
- return FilterBuilders.missingFilter(name);
+ return QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery((name)));
case "contains":
checkRequiredValue(expectedValue, name, op, false);
- return FilterBuilders.regexpFilter(name, ".*" + expectedValue + ".*");
+ return QueryBuilders.regexpQuery(name, ".*" + expectedValue + ".*");
case "startsWith":
checkRequiredValue(expectedValue, name, op, false);
- return FilterBuilders.prefixFilter(name, expectedValue);
+ return QueryBuilders.prefixQuery(name, expectedValue);
case "endsWith":
checkRequiredValue(expectedValue, name, op, false);
- return FilterBuilders.regexpFilter(name, ".*" + expectedValue);
+ return QueryBuilders.regexpQuery(name, ".*" + expectedValue);
case "matchesRegex":
checkRequiredValue(expectedValue, name, op, false);
- return FilterBuilders.regexpFilter(name, expectedValue);
+ return QueryBuilders.regexpQuery(name, expectedValue);
case "in":
checkRequiredValue(values, name, op, true);
- return FilterBuilders.inFilter(name, values.toArray());
+ return QueryBuilders.termsQuery(name, values.toArray());
case "notIn":
checkRequiredValue(values, name, op, true);
- return FilterBuilders.notFilter(FilterBuilders.inFilter(name, values.toArray()));
+ return QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(name, values.toArray()));
case "all":
checkRequiredValue(values, name, op, true);
- return FilterBuilders.termsFilter(name, values.toArray()).execution("and");
+ BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+ for (Object curValue : values) {
+ boolQueryBuilder.must(QueryBuilders.termQuery(name, curValue));
+ }
+ return boolQueryBuilder;
case "isDay":
checkRequiredValue(value, name, op, false);
return getIsSameDayRange(value, name);
case "isNotDay":
checkRequiredValue(value, name, op, false);
- return FilterBuilders.notFilter(getIsSameDayRange(value, name));
+ return QueryBuilders.boolQuery().mustNot(getIsSameDayRange(value, name));
}
return null;
}
@@ -126,10 +131,10 @@ public class PropertyConditionESQueryBuilder implements ConditionESQueryBuilder
}
}
- private FilterBuilder getIsSameDayRange (Object value, String name) {
+ private QueryBuilder getIsSameDayRange (Object value, String name) {
DateTime date = new DateTime(value);
DateTime dayStart = date.withTimeAtStartOfDay();
DateTime dayAfterStart = date.plusDays(1).withTimeAtStartOfDay();
- return FilterBuilders.rangeFilter(name).gte(dayStart.toDate()).lte(dayAfterStart.toDate());
+ return QueryBuilders.rangeQuery(name).gte(dayStart.toDate()).lte(dayAfterStart.toDate());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
index ca3e512..e0c5e61 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
@@ -41,7 +41,6 @@ import org.slf4j.LoggerFactory;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
/**
@@ -271,7 +270,7 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
if (value instanceof Date) {
return ((Date) value);
} else {
- DateMathParser parser = new DateMathParser(DateFieldMapper.Defaults.DATE_TIME_FORMATTER, TimeUnit.MILLISECONDS);
+ DateMathParser parser = new DateMathParser(DateFieldMapper.Defaults.DATE_TIME_FORMATTER);
try {
return new Date(parser.parse(value.toString(), new Callable<Long>() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/SourceEventPropertyConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/SourceEventPropertyConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/SourceEventPropertyConditionESQueryBuilder.java
index 68e5944..7ba6c6f 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/SourceEventPropertyConditionESQueryBuilder.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/SourceEventPropertyConditionESQueryBuilder.java
@@ -20,8 +20,9 @@ package org.apache.unomi.plugins.baseplugin.conditions;
import org.apache.unomi.api.conditions.Condition;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
import java.util.ArrayList;
import java.util.List;
@@ -32,21 +33,29 @@ public class SourceEventPropertyConditionESQueryBuilder implements ConditionESQu
public SourceEventPropertyConditionESQueryBuilder() {
}
- private void appendFilderIfPropExist(List<FilterBuilder> filterBuilders, Condition condition, String prop){
+ private void appendFilderIfPropExist(List<QueryBuilder> queryBuilders, Condition condition, String prop){
final Object parameter = condition.getParameter(prop);
if (parameter != null && !"".equals(parameter)) {
- filterBuilders.add(FilterBuilders.termFilter("source." + prop, (String) parameter));
+ queryBuilders.add(QueryBuilders.termQuery("source." + prop, (String) parameter));
}
}
- public FilterBuilder buildFilter(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher) {
- List<FilterBuilder> l = new ArrayList<FilterBuilder>();
+ public QueryBuilder buildQuery(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher) {
+ List<QueryBuilder> queryBuilders = new ArrayList<QueryBuilder>();
for (String prop : new String[]{"id", "path", "scope", "type"}){
- appendFilderIfPropExist(l, condition, prop);
+ appendFilderIfPropExist(queryBuilders, condition, prop);
}
- if (l.size() >= 1) {
- return l.size() == 1 ? l.get(0) : FilterBuilders.andFilter(l.toArray(new FilterBuilder[l.size()]));
+ if (queryBuilders.size() >= 1) {
+ if (queryBuilders.size() == 1) {
+ return queryBuilders.get(0);
+ } else {
+ BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+ for (QueryBuilder queryBuilder : queryBuilders) {
+ boolQueryBuilder.must(queryBuilder);
+ }
+ return boolQueryBuilder;
+ }
} else {
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/plugins/hover-event/src/main/java/org/apache/unomi/plugins/events/hover/querybuilders/HoverEventConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/plugins/hover-event/src/main/java/org/apache/unomi/plugins/events/hover/querybuilders/HoverEventConditionESQueryBuilder.java b/plugins/hover-event/src/main/java/org/apache/unomi/plugins/events/hover/querybuilders/HoverEventConditionESQueryBuilder.java
index 201b834..978edef 100644
--- a/plugins/hover-event/src/main/java/org/apache/unomi/plugins/events/hover/querybuilders/HoverEventConditionESQueryBuilder.java
+++ b/plugins/hover-event/src/main/java/org/apache/unomi/plugins/events/hover/querybuilders/HoverEventConditionESQueryBuilder.java
@@ -20,8 +20,9 @@ package org.apache.unomi.plugins.events.hover.querybuilders;
import org.apache.unomi.api.conditions.Condition;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
import java.util.ArrayList;
import java.util.List;
@@ -35,19 +36,23 @@ public class HoverEventConditionESQueryBuilder implements ConditionESQueryBuilde
public HoverEventConditionESQueryBuilder() {
}
- public FilterBuilder buildFilter(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher) {
- List<FilterBuilder> filters = new ArrayList<FilterBuilder>();
- filters.add(FilterBuilders.termFilter("eventType", "hover"));
+ public QueryBuilder buildQuery(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher) {
+ List<QueryBuilder> queryBuilders = new ArrayList<QueryBuilder>();
+ queryBuilders.add(QueryBuilders.termQuery("eventType", "hover"));
String targetId = (String) condition.getParameter("targetId");
String targetPath = (String) condition.getParameter("targetPath");
if (targetId != null && targetId.trim().length() > 0) {
- filters.add(FilterBuilders.termFilter("target.itemId", targetId));
+ queryBuilders.add(QueryBuilders.termQuery("target.itemId", targetId));
} else if (targetPath != null && targetPath.trim().length() > 0) {
- filters.add(FilterBuilders.termFilter("target.properties.pageInfo.pagePath", targetPath));
+ queryBuilders.add(QueryBuilders.termQuery("target.properties.pageInfo.pagePath", targetPath));
} else {
- filters.add(FilterBuilders.termFilter("target.itemId", ""));
+ queryBuilders.add(QueryBuilders.termQuery("target.itemId", ""));
}
- return FilterBuilders.andFilter(filters.toArray(new FilterBuilder[filters.size()]));
+ BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+ for (QueryBuilder queryBuilder : queryBuilders) {
+ boolQueryBuilder.must(queryBuilder);
+ }
+ return boolQueryBuilder;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index db4e1dd..a94c0a0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -65,11 +65,11 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<cxf.version>2.7.11</cxf.version>
- <version.jackson.core>2.4.0</version.jackson.core> <!-- version used by CXF -->
- <version.jackson.jaxb>2.4.0</version.jackson.jaxb> <!-- version used by CXF -->
+ <version.jackson.core>2.6.6</version.jackson.core> <!-- version used by CXF -->
+ <version.jackson.jaxb>2.6.6</version.jackson.jaxb> <!-- version used by CXF -->
<version.karaf>3.0.2</version.karaf>
- <version.pax.exam>4.4.0</version.pax.exam>
- <elasticsearch.version>1.6.2</elasticsearch.version>
+ <version.pax.exam>4.9.1</version.pax.exam>
+ <elasticsearch.version>2.3.4</elasticsearch.version>
<maven.compiler.source>1.7</maven.compiler.source>
<maven.compiler.target>1.7</maven.compiler.target>
[2/2] incubator-unomi git commit: UNOMI-28 Upgrade to ElasticSearch
2.x
Posted by sh...@apache.org.
UNOMI-28 Upgrade to ElasticSearch 2.x
Used this resource to perform changes:
https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking_20_java_api_changes.html
A lot of changes are contained in this commit, notably :
- Moved data directory from KARAF_HOME/data to KARAF_HOME/elasticsearch/data
- Had to build a JSON parser to rewrite field names with "." to escape them with "__DOT__"
- Fixes on dates
- Replace all filter classes with query classes as these don't exist in ES 2.x any more
- Replace and/or ES clauses with bool must/should
- Had to replace deleteByQuery support to add plugin by default, since in ES 2.x they externalized the functionality
- Had to install Groovy and Expression modules since they are no longer part of the core
What is NOT done by this patch is the migration procedure
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/131a06aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/131a06aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/131a06aa
Branch: refs/heads/UNOMI-28-ES-2-X-UPGRADE
Commit: 131a06aa78fc86223359c11c7a40a4e653dab434
Parents: 1c5d74a
Author: Serge Huber <sh...@apache.org>
Authored: Tue Aug 9 15:55:03 2016 +0200
Committer: Serge Huber <sh...@apache.org>
Committed: Tue Aug 9 15:55:03 2016 +0200
----------------------------------------------------------------------
.../META-INF/cxs/mappings/geonameEntry.json | 2 +-
itests/pom.xml | 5 +-
.../java/org/apache/unomi/itests/AllTests.java | 11 +-
.../apache/unomi/itests/ProfileServiceTest.java | 53 +++
kar/src/main/feature/feature.xml | 12 +-
package/pom.xml | 54 ++-
persistence-elasticsearch/core/pom.xml | 207 +++-------
.../ElasticSearchPersistenceServiceImpl.java | 165 +++++---
.../elasticsearch/FieldDotEscaper.java | 108 ++++++
.../elasticsearch/FieldDotJsonTransformer.java | 290 ++++++++++++++
.../conditions/ConditionContextHelper.java | 4 +-
.../conditions/ConditionESQueryBuilder.java | 4 +-
.../ConditionESQueryBuilderDispatcher.java | 16 +-
.../META-INF/cxs/mappings/campaign.json | 4 +-
.../META-INF/cxs/mappings/campaignevent.json | 2 +-
.../resources/META-INF/cxs/mappings/event.json | 2 +-
.../META-INF/cxs/mappings/profile.json | 2 +-
.../META-INF/cxs/mappings/session.json | 4 +-
.../core/src/main/resources/elasticsearch.yml | 381 +++----------------
.../elasticsearch/FieldDotEscapeTest.java | 95 +++++
.../FieldDotJsonTransformerTest.java | 61 +++
.../core/src/test/resources/complex.json | 50 +++
.../core/src/test/resources/profile.json | 1 +
.../plugin/security/SecurityPlugin.java | 14 +-
.../plugin/security/SecurityPluginModule.java | 3 +-
.../plugin/security/SecurityPluginService.java | 4 +-
.../BooleanConditionESQueryBuilder.java | 20 +-
...onByPointSessionConditionESQueryBuilder.java | 10 +-
.../MatchAllConditionESQueryBuilder.java | 8 +-
.../conditions/NotConditionESQueryBuilder.java | 8 +-
.../PastEventConditionESQueryBuilder.java | 8 +-
.../PropertyConditionESQueryBuilder.java | 51 +--
.../conditions/PropertyConditionEvaluator.java | 3 +-
...rceEventPropertyConditionESQueryBuilder.java | 27 +-
.../HoverEventConditionESQueryBuilder.java | 23 +-
pom.xml | 8 +-
36 files changed, 1044 insertions(+), 676 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json
----------------------------------------------------------------------
diff --git a/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json b/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json
index 6ccf39b..63664fe 100644
--- a/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json
+++ b/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json
@@ -58,7 +58,7 @@
},
"modificationDate": {
"type": "date",
- "format": "dateOptionalTime"
+ "format": "strict_date_optional_time||epoch_millis"
},
"name": {
"type": "string",
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/itests/pom.xml
----------------------------------------------------------------------
diff --git a/itests/pom.xml b/itests/pom.xml
index b156a45..9c2c00b 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -106,7 +106,10 @@
<artifactId>javax.inject</artifactId>
<scope>test</scope>
</dependency>
-
+ <dependency>
+ <groupId>org.osgi</groupId>
+ <artifactId>org.osgi.core</artifactId>
+ </dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/itests/src/test/java/org/apache/unomi/itests/AllTests.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/AllTests.java b/itests/src/test/java/org/apache/unomi/itests/AllTests.java
index 8f017ba..0fbd383 100644
--- a/itests/src/test/java/org/apache/unomi/itests/AllTests.java
+++ b/itests/src/test/java/org/apache/unomi/itests/AllTests.java
@@ -27,11 +27,12 @@ import org.junit.runners.Suite.SuiteClasses;
* @author Sergiy Shyrkov
*/
@RunWith(Suite.class)
-@SuiteClasses({
- //BasicTest.class,
- ConditionEvaluatorTest.class,
- ConditionESQueryBuilderTest.class,
- SegmentTest.class
+@SuiteClasses({
+ //BasicTest.class,
+ ConditionEvaluatorTest.class,
+ ConditionESQueryBuilderTest.class,
+ SegmentTest.class,
+ ProfileServiceTest.class
})
public class AllTests {
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/itests/src/test/java/org/apache/unomi/itests/ProfileServiceTest.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileServiceTest.java b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceTest.java
new file mode 100644
index 0000000..4f5dba8
--- /dev/null
+++ b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+package org.apache.unomi.itests;
+
+import org.apache.unomi.api.Profile;
+import org.apache.unomi.api.services.ProfileService;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.ops4j.pax.exam.junit.PaxExam;
+import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy;
+import org.ops4j.pax.exam.spi.reactors.PerSuite;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+
+/**
+ * An integration test for the profile service
+ */
+@RunWith(PaxExam.class)
+@ExamReactorStrategy(PerSuite.class)
+public class ProfileServiceTest extends BaseTest {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(SegmentTest.class);
+ private final static String TEST_PROFILE_ID = "test-profile-id";
+ @Inject
+ protected ProfileService profileService;
+
+ @Test
+ public void testProfileDelete() {
+ Profile profile = new Profile();
+ profile.setItemId(TEST_PROFILE_ID);
+ profileService.save(profile);
+ LOGGER.info("Profile saved, now testing profile delete...");
+ profileService.delete(TEST_PROFILE_ID, false);
+ LOGGER.info("Profile deleted successfully.");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/kar/src/main/feature/feature.xml
----------------------------------------------------------------------
diff --git a/kar/src/main/feature/feature.xml b/kar/src/main/feature/feature.xml
index 5e63b8b..0a66af8 100644
--- a/kar/src/main/feature/feature.xml
+++ b/kar/src/main/feature/feature.xml
@@ -31,12 +31,12 @@
<configfile finalname="/etc/org.apache.unomi.geonames.cfg">mvn:org.apache.unomi/cxs-geonames-services/${project.version}/cfg/geonamescfg</configfile>
<configfile finalname="/etc/elasticsearch.yml">mvn:org.apache.unomi/unomi-persistence-elasticsearch-core/${project.version}/yml/elasticsearchconfig</configfile>
<bundle start-level="75">mvn:commons-io/commons-io/2.4</bundle>
- <bundle start-level="75">mvn:com.fasterxml.jackson.core/jackson-core/2.4.0</bundle>
- <bundle start-level="75">mvn:com.fasterxml.jackson.core/jackson-databind/2.4.0</bundle>
- <bundle start-level="75">mvn:com.fasterxml.jackson.core/jackson-annotations/2.4.0</bundle>
- <bundle start-level="75">mvn:com.fasterxml.jackson.jaxrs/jackson-jaxrs-base/2.4.0</bundle>
- <bundle start-level="75">mvn:com.fasterxml.jackson.module/jackson-module-jaxb-annotations/2.4.0</bundle>
- <bundle start-level="75">mvn:com.fasterxml.jackson.jaxrs/jackson-jaxrs-json-provider/2.4.0</bundle>
+ <bundle start-level="75">mvn:com.fasterxml.jackson.core/jackson-core/${version.jackson.core}</bundle>
+ <bundle start-level="75">mvn:com.fasterxml.jackson.core/jackson-databind/${version.jackson.core}</bundle>
+ <bundle start-level="75">mvn:com.fasterxml.jackson.core/jackson-annotations/${version.jackson.core}</bundle>
+ <bundle start-level="75">mvn:com.fasterxml.jackson.jaxrs/jackson-jaxrs-base/${version.jackson.core}</bundle>
+ <bundle start-level="75">mvn:com.fasterxml.jackson.module/jackson-module-jaxb-annotations/${version.jackson.jaxb}</bundle>
+ <bundle start-level="75">mvn:com.fasterxml.jackson.jaxrs/jackson-jaxrs-json-provider/${version.jackson.core}</bundle>
<bundle start-level="75">mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.jakarta-regexp/1.4_1</bundle>
<bundle start-level="75">mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.joda-time/2.3_1</bundle>
<bundle start-level="75">mvn:org.apache.unomi/unomi-api/${project.version}</bundle>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/package/pom.xml
----------------------------------------------------------------------
diff --git a/package/pom.xml b/package/pom.xml
index eb6546b..c55aed0 100644
--- a/package/pom.xml
+++ b/package/pom.xml
@@ -122,7 +122,7 @@
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
- <id>unpack-sigar</id>
+ <id>unpack-deploy-by-query-plugin</id>
<phase>generate-resources</phase>
<goals>
<goal>unpack</goal>
@@ -130,12 +130,32 @@
<configuration>
<artifactItems>
<artifactItem>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
+ <groupId>org.elasticsearch.plugin</groupId>
+ <artifactId>delete-by-query</artifactId>
<version>${elasticsearch.version}</version>
- <type>tar.gz</type>
- <includes>**/lib/sigar/**</includes>
- <outputDirectory>${project.build.directory}/assembly</outputDirectory>
+ <type>zip</type>
+ <outputDirectory>${project.build.directory}/assembly/elasticsearch/plugins/delete-by-query</outputDirectory>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.elasticsearch.module</groupId>
+ <artifactId>lang-groovy</artifactId>
+ <version>${elasticsearch.version}</version>
+ <type>zip</type>
+ <outputDirectory>${project.build.directory}/assembly/elasticsearch/modules/lang-groovy</outputDirectory>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.elasticsearch.module</groupId>
+ <artifactId>lang-expression</artifactId>
+ <version>${elasticsearch.version}</version>
+ <type>zip</type>
+ <outputDirectory>${project.build.directory}/assembly/elasticsearch/modules/lang-expression</outputDirectory>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.elasticsearch.module</groupId>
+ <artifactId>reindex</artifactId>
+ <version>${elasticsearch.version}</version>
+ <type>zip</type>
+ <outputDirectory>${project.build.directory}/assembly/elasticsearch/modules/reindex</outputDirectory>
</artifactItem>
</artifactItems>
</configuration>
@@ -210,28 +230,6 @@
</executions>
</plugin>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-karaf</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <move todir="${project.build.directory}/assembly/lib/sigar">
- <fileset dir="${project.build.directory}/assembly/elasticsearch-${elasticsearch.version}/lib/sigar" />
- </move>
- <delete dir="${project.build.directory}/assembly/elasticsearch-${elasticsearch.version}" />
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
<groupId>org.apache.karaf.tooling</groupId>
<artifactId>karaf-maven-plugin</artifactId>
<executions>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/pom.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/pom.xml b/persistence-elasticsearch/core/pom.xml
index eddadf1..e44bd7a 100644
--- a/persistence-elasticsearch/core/pom.xml
+++ b/persistence-elasticsearch/core/pom.xml
@@ -52,101 +52,64 @@
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>${elasticsearch.version}</version>
- </dependency>
-
- <!-- The following are optional dependencies from the ElasticSearch that are made mandatory -->
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-expressions</artifactId>
- <version>4.10.2</version>
- </dependency>
- <dependency>
- <groupId>org.antlr</groupId>
- <artifactId>antlr-runtime</artifactId>
- <version>3.5</version>
<scope>compile</scope>
- <exclusions>
- <exclusion>
- <artifactId>stringtemplate</artifactId>
- <groupId>org.antlr</groupId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- <version>4.1</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm-commons</artifactId>
- <version>4.1</version>
+ <groupId>org.elasticsearch.plugin</groupId>
+ <artifactId>delete-by-query</artifactId>
+ <version>${elasticsearch.version}</version>
<scope>compile</scope>
- <exclusions>
- <exclusion>
- <artifactId>asm-tree</artifactId>
- <groupId>org.ow2.asm</groupId>
- </exclusion>
- </exclusions>
</dependency>
+
+ <!-- The following are optional dependencies from the ElasticSearch that are made mandatory -->
+
<dependency>
- <groupId>com.spatial4j</groupId>
- <artifactId>spatial4j</artifactId>
- <version>0.4.1</version>
- </dependency>
- <!--dependency>
- <groupId>com.vividsolutions</groupId>
- <artifactId>jts</artifactId>
- <version>1.13</version>
- </dependency-->
- <dependency>
- <groupId>org.codehaus.groovy</groupId>
- <artifactId>groovy-all</artifactId>
- <version>2.3.2</version>
- </dependency>
- <!--
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>1.2.17</version>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <scope>provided</scope>
</dependency>
- -->
+
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
<version>1.7.7</version>
+ <scope>compile</scope>
</dependency>
- <!--
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>1.6.2</version>
- </dependency>
- -->
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
<version>4.1.0</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.github.spullara.mustache.java</groupId>
+ <artifactId>compiler</artifactId>
+ <version>0.8.13</version>
+ <scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.fusesource</groupId>
- <artifactId>sigar</artifactId>
- <version>1.6.4</version>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>2.8.1</version>
+ <scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.fusesource</groupId>
- <artifactId>sigar</artifactId>
- <version>1.6.4</version>
- <classifier>native</classifier>
+ <groupId>com.twitter</groupId>
+ <artifactId>util-core</artifactId>
+ <version>4.0.0</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.noggit</groupId>
+ <artifactId>noggit</artifactId>
+ <version>0.7</version>
+ <scope>compile</scope>
</dependency>
-
<!-- End of optional ElasticSearch dependencies -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
@@ -161,6 +124,18 @@
<groupId>org.mvel</groupId>
<artifactId>mvel2</artifactId>
</dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<build>
@@ -173,104 +148,30 @@
<instructions>
<Import-Package>
com.google.protobuf;resolution:=optional,
- com.ibm.uvm.tools;resolution:=optional,
- com.sun.jdmk.comm;resolution:=optional,
- com.thoughtworks.qdox;resolution:=optional,
- com.thoughtworks.qdox.model;resolution:=optional,
- com.thoughtworks.xstream;resolution:=optional,
com.vividsolutions.jts.*;resolution:=optional,
- groovyjarjarasm.asm.tree.analysis;resolution:=optional,
- javax.accessibility;resolution:=optional,
javax.annotation;resolution:=optional,
- javax.jmdns;resolution:=optional,
- javax.jms;resolution:=optional,
- javax.mail;resolution:=optional,
- javax.mail.internet;resolution:=optional,
- javax.management;resolution:=optional,
- javax.management.modelmbean;resolution:=optional,
- javax.management.remote;resolution:=optional,
- javax.management.remote.rmi;resolution:=optional,
- javax.management.timer;resolution:=optional,
- javax.naming;resolution:=optional,
+ javax.crypto;resolution:=optional,
+ javax.crypto.spec;resolution:=optional,
javax.net.ssl;resolution:=optional,
- javax.print.attribute.standard;resolution:=optional,
- javax.rmi.ssl;resolution:=optional,
- javax.script;resolution:=optional,
+ javax.security.auth.x500;resolution:=optional,
+ javax.security.cert;resolution:=optional,
javax.servlet;resolution:=optional,
javax.servlet.http;resolution:=optional,
- javax.servlet.jsp;resolution:=optional,
- javax.sql;resolution:=optional,
- javax.swing;resolution:=optional,
- javax.swing.border;resolution:=optional,
- javax.swing.event;resolution:=optional,
- javax.swing.filechooser;resolution:=optional,
- javax.swing.plaf;resolution:=optional,
- javax.swing.plaf.metal;resolution:=optional,
- javax.swing.table;resolution:=optional,
- javax.swing.text;resolution:=optional,
- javax.swing.text.html;resolution:=optional,
- javax.swing.tree;resolution:=optional,
- javax.swing.undo;resolution:=optional,
- javax.xml.namespace;resolution:=optional,
+ javax.xml.bind;resolution:=optional,
javax.xml.parsers;resolution:=optional,
javax.xml.transform;resolution:=optional,
javax.xml.transform.dom;resolution:=optional,
javax.xml.transform.stream;resolution:=optional,
- javax.xml.validation;resolution:=optional,
- javax.xml.xpath;resolution:=optional,
- jline;resolution:=optional,
- jline.console;resolution:=optional,
- jline.console.completer;resolution:=optional,
- jline.console.history;resolution:=optional,
- jline.internal;resolution:=optional,
- junit.framework; resolution:=optional
- junit.framework;resolution:=optional,
- junit.textui; resolution:=optional
- junit.textui;resolution:=optional,
- org.antlr.stringtemplate;resolution:=optional,
- org.apache.bsf;resolution:=optional,
- org.apache.bsf.util;resolution:=optional,
- org.apache.commons.cli;resolution:=optional,
org.apache.commons.logging;resolution:=optional,
- org.apache.ivy;resolution:=optional,
- org.apache.ivy.core.cache;resolution:=optional,
- org.apache.ivy.core.event;resolution:=optional,
- org.apache.ivy.core.event.download;resolution:=optional,
- org.apache.ivy.core.event.resolve;resolution:=optional,
- org.apache.ivy.core.module.descriptor;resolution:=optional,
- org.apache.ivy.core.module.id;resolution:=optional,
- org.apache.ivy.core.report;resolution:=optional,
- org.apache.ivy.core.resolve;resolution:=optional,
- org.apache.ivy.core.settings;resolution:=optional,
- org.apache.ivy.plugins.matcher;resolution:=optional,
- org.apache.ivy.plugins.resolver;resolution:=optional,
- org.apache.ivy.util;resolution:=optional,
- org.apache.log;resolution:=optional,
- org.apache.regexp; resolution:=optional
org.apache.regexp;resolution:=optional,
- org.apache.tomcat.jni; resolution:=optional,
- org.apache.tools.ant;resolution:=optional,
- org.apache.tools.ant.dispatch;resolution:=optional,
- org.apache.tools.ant.helper;resolution:=optional,
- org.apache.tools.ant.taskdefs; resolution:=optional
- org.apache.tools.ant.taskdefs;resolution:=optional,
- org.apache.tools.ant.types;resolution:=optional,
- org.apache.tools.ant.util;resolution:=optional,
- org.bouncycastle.asn1.x500;resolution:=optional,
- org.bouncycastle.cert;resolution:=optional,
- org.bouncycastle.cert.jcajce;resolution:=optional,
- org.bouncycastle.jce.provider;resolution:=optional,
- org.bouncycastle.operator;resolution:=optional,
- org.bouncycastle.operator.jcajce;resolution:=optional,
+ org.apache.tomcat.jni;resolution:=optional,
+ org.bouncycastle.*;resolution:=optional,
org.eclipse.jetty.npn;resolution:=optional,
- org.fusesource.jansi;resolution:=optional,
org.jboss.logging;resolution:=optional,
org.jboss.marshalling;resolution:=optional,
- org.junit;resolution:=optional,
- org.objectweb.asm.tree;resolution:=optional,
- sun.awt;resolution:=optional,
- sun.java2d.pipe;resolution:=optional,
- sun.misc; resolution:=optional,
+ org.jruby;resolution:=optional,
+ org.jruby.embed;resolution:=optional,
+ sun.misc;resolution:=optional,
sun.security.util;resolution:=optional,
sun.security.x509;resolution:=optional,
*
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 7bf2f14..743def9 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -17,6 +17,7 @@
package org.apache.unomi.persistence.elasticsearch;
+import com.google.common.collect.UnmodifiableIterator;
import org.apache.unomi.api.ClusterNode;
import org.apache.unomi.api.Item;
import org.apache.unomi.api.PartialList;
@@ -31,6 +32,7 @@ import org.apache.unomi.persistence.spi.CustomObjectMapper;
import org.apache.unomi.persistence.spi.PersistenceService;
import org.apache.unomi.persistence.spi.aggregate.*;
import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
+import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
@@ -40,26 +42,29 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.deletebyquery.DeleteByQueryAction;
+import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder;
+import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.percolate.PercolateResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.action.support.nodes.NodesOperationRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.collect.UnmodifiableIterator;
-import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsException;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.query.*;
-import org.elasticsearch.indices.IndexMissingException;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.node.Node;
+import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
@@ -68,8 +73,8 @@ import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregation;
import org.elasticsearch.search.aggregations.bucket.filter.Filter;
import org.elasticsearch.search.aggregations.bucket.global.Global;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogram;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.missing.MissingBuilder;
import org.elasticsearch.search.aggregations.bucket.range.RangeBuilder;
import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeBuilder;
@@ -86,10 +91,13 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
+import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
+import java.nio.file.Paths;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
@@ -99,8 +107,32 @@ import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
@SuppressWarnings("rawtypes")
public class ElasticSearchPersistenceServiceImpl implements PersistenceService, ClusterService, SynchronousBundleListener {
- public static final long MILLIS_PER_DAY = 24L * 60L * 60L * 1000L;
private static final Logger logger = LoggerFactory.getLogger(ElasticSearchPersistenceServiceImpl.class.getName());
+
+ public static final String DISCOVERY_ZEN_PING_MULTICAST_ENABLED = "discovery.zen.ping.multicast.enabled";
+ public static final String CONTEXTSERVER_ADDRESS = "contextserver.address";
+ public static final String CONTEXTSERVER_PORT = "contextserver.port";
+ public static final String CONTEXTSERVER_SECURE_ADDRESS = "contextserver.secureAddress";
+ public static final String CONTEXTSERVER_SECURE_PORT = "contextserver.securePort";
+ public static final String KARAF_HOME = "karaf.home";
+ public static final String ELASTICSEARCH_HOME_DIRECTORY = "elasticsearch";
+ public static final String ELASTICSEARCH_PLUGINS_DIRECTORY = ELASTICSEARCH_HOME_DIRECTORY + "/plugins";
+ public static final String ELASTICSEARCH_DATA_DIRECTORY = ELASTICSEARCH_HOME_DIRECTORY + "/data";
+ public static final String INDEX_NUMBER_OF_REPLICAS = "index.number_of_replicas";
+ public static final String INDEX_NUMBER_OF_SHARDS = "index.number_of_shards";
+ public static final String NODE_CONTEXTSERVER_ADDRESS = "node.contextserver.address";
+ public static final String NODE_CONTEXTSERVER_PORT = "node.contextserver.port";
+ public static final String NODE_CONTEXTSERVER_SECURE_ADDRESS = "node.contextserver.secureAddress";
+ public static final String NODE_CONTEXTSERVER_SECURE_PORT = "node.contextserver.securePort";
+ public static final String NUMBER_OF_SHARDS = "number_of_shards";
+ public static final String NUMBER_OF_REPLICAS = "number_of_replicas";
+ public static final String CLUSTER_NAME = "cluster.name";
+ public static final String NODE_DATA = "node.data";
+ public static final String PATH_DATA = "path.data";
+ public static final String PATH_HOME = "path.home";
+ public static final String PATH_PLUGINS = "path.plugins";
+ public static final String INDEX_MAX_RESULT_WINDOW = "index.max_result_window";
+
private Node node;
private Client client;
private String clusterName;
@@ -220,12 +252,12 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
Map<String, String> settings = null;
if (elasticSearchConfig != null && elasticSearchConfig.length() > 0) {
try {
- URL elasticSearchConfigURL = new URL(elasticSearchConfig);
- Settings.Builder settingsBuilder = ImmutableSettings.builder().loadFromUrl(elasticSearchConfigURL);
+ URI elasticSearchConfigURI = new URI(elasticSearchConfig);
+ Settings.Builder settingsBuilder = Settings.builder().loadFromPath(Paths.get(elasticSearchConfigURI));
settings = settingsBuilder.build().getAsMap();
- logger.info("Successfully loaded ElasticSearch configuration from " + elasticSearchConfigURL);
- } catch (MalformedURLException e) {
- logger.error("Error in ElasticSearch configuration URL ", e);
+ logger.info("Successfully loaded ElasticSearch configuration from " + elasticSearchConfigURI);
+ } catch (URISyntaxException e) {
+ logger.error("Error in ElasticSearch configuration URI ", e);
} catch (SettingsException se) {
logger.info("Error trying to load settings from " + elasticSearchConfig + ": " + se.getMessage() + " (activate debug mode for exception details)");
if (logger.isDebugEnabled()) {
@@ -234,25 +266,34 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
}
- address = System.getProperty("contextserver.address", address);
- port = System.getProperty("contextserver.port", port);
- secureAddress = System.getProperty("contextserver.secureAddress", secureAddress);
- securePort = System.getProperty("contextserver.securePort", securePort);
+ address = System.getProperty(CONTEXTSERVER_ADDRESS, address);
+ port = System.getProperty(CONTEXTSERVER_PORT, port);
+ secureAddress = System.getProperty(CONTEXTSERVER_SECURE_ADDRESS, secureAddress);
+ securePort = System.getProperty(CONTEXTSERVER_SECURE_PORT, securePort);
- ImmutableSettings.Builder settingsBuilder = ImmutableSettings.builder();
+ Settings.Builder settingsBuilder = Settings.builder();
if (settings != null) {
settingsBuilder.put(settings);
}
- settingsBuilder.put("cluster.name", clusterName)
- .put("node.data", nodeData)
- .put("discovery.zen.ping.multicast.enabled", discoveryEnabled)
- .put("index.number_of_replicas", numberOfReplicas)
- .put("index.number_of_shards", numberOfShards)
- .put("node.contextserver.address", address)
- .put("node.contextserver.port", port)
- .put("node.contextserver.secureAddress", secureAddress)
- .put("node.contextserver.securePort", securePort);
+ String karafHome = System.getProperty(KARAF_HOME);
+ File pluginsFile = new File(getConfig(settings, PATH_PLUGINS, new File(new File(karafHome), ELASTICSEARCH_PLUGINS_DIRECTORY).getAbsolutePath()));
+ File homeFile = new File(getConfig(settings, PATH_HOME, new File(new File(karafHome), ELASTICSEARCH_HOME_DIRECTORY).getAbsolutePath()));
+ File dataFile = new File(getConfig(settings, PATH_DATA, new File(new File(karafHome), ELASTICSEARCH_DATA_DIRECTORY).getAbsolutePath()));
+
+ settingsBuilder.put(CLUSTER_NAME, clusterName)
+ .put(NODE_DATA, nodeData)
+ .put(PATH_DATA, dataFile.getAbsolutePath())
+ .put(PATH_HOME, homeFile.getAbsolutePath())
+ .put(PATH_PLUGINS, pluginsFile.getAbsolutePath())
+ .put(DISCOVERY_ZEN_PING_MULTICAST_ENABLED, discoveryEnabled)
+ .put(INDEX_NUMBER_OF_REPLICAS, numberOfReplicas)
+ .put(INDEX_NUMBER_OF_SHARDS, numberOfShards)
+ .put(NODE_CONTEXTSERVER_ADDRESS, address)
+ .put(NODE_CONTEXTSERVER_PORT, port)
+ .put(NODE_CONTEXTSERVER_SECURE_ADDRESS, secureAddress)
+ .put(NODE_CONTEXTSERVER_SECURE_PORT, securePort)
+ .put(INDEX_MAX_RESULT_WINDOW, "2147483647");
node = nodeBuilder().settings(settingsBuilder).node();
client = node.client();
@@ -284,9 +325,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
client.admin().indices().preparePutTemplate(indexName + "_monthlyindex")
.setTemplate(indexName + "-*")
.setOrder(1)
- .setSettings(ImmutableSettings.settingsBuilder()
- .put("number_of_shards", Integer.parseInt(monthlyIndexNumberOfShards))
- .put("number_of_replicas", Integer.parseInt(monthlyIndexNumberOfReplicas))
+ .setSettings(Settings.settingsBuilder()
+ .put(NUMBER_OF_SHARDS, Integer.parseInt(monthlyIndexNumberOfShards))
+ .put(NUMBER_OF_REPLICAS, Integer.parseInt(monthlyIndexNumberOfReplicas))
.build()).execute().actionGet();
getMonthlyIndex(new Date(), true);
@@ -442,7 +483,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
@Override
public long getAllItemsCount(String itemType) {
- return queryCount(FilterBuilders.matchAllFilter(), itemType);
+ return queryCount(QueryBuilders.matchAllQuery(), itemType);
}
@Override
@@ -477,19 +518,19 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
.actionGet();
if (response.isExists()) {
String sourceAsString = response.getSourceAsString();
- final T value = CustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz);
+ final T value = CustomObjectMapper.getObjectMapper().readValue(FieldDotEscaper.unescapeJson(sourceAsString), clazz);
value.setItemId(response.getId());
return value;
} else {
return null;
}
}
- } catch (IndexMissingException e) {
- logger.debug("No index found for itemType=" + clazz.getName() + "itemId=" + itemId, e);
+ } catch (IndexNotFoundException e) {
+ logger.debug("No index found for itemType=" + clazz.getName() + " itemId=" + itemId, e);
} catch (IllegalAccessException e) {
- logger.error("Error loading itemType=" + clazz.getName() + "itemId=" + itemId, e);
+ logger.error("Error loading itemType=" + clazz.getName() + " itemId=" + itemId, e);
} catch (Exception t) {
- logger.error("Error loading itemType=" + clazz.getName() + "itemId=" + itemId, t);
+ logger.error("Error loading itemType=" + clazz.getName() + " itemId=" + itemId, t);
}
return null;
}
@@ -504,6 +545,11 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
protected Boolean execute(Object... args) {
try {
String source = CustomObjectMapper.getObjectMapper().writeValueAsString(item);
+ Set<String> modifiedNames = new LinkedHashSet<>();
+ source = FieldDotEscaper.escapeJson(source, modifiedNames);
+ if (modifiedNames.size() > 0) {
+ logger.warn("Found JSON property names with dot characters not allowed by ElasticSearch 2.x={} in item {}", modifiedNames, item);
+ }
String itemType = item.getItemType();
String index = indexNames.containsKey(itemType) ? indexNames.get(itemType) :
(itemsMonthlyIndexed.contains(itemType) ? getMonthlyIndex(((TimestampedItem) item).getTimeStamp()) : indexName);
@@ -514,7 +560,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
try {
indexBuilder.execute().actionGet();
- } catch (IndexMissingException e) {
+ } catch (IndexNotFoundException e) {
if (itemsMonthlyIndexed.contains(itemType)) {
getMonthlyIndex(((TimestampedItem) item).getTimeStamp(), true);
indexBuilder.execute().actionGet();
@@ -545,11 +591,11 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
String index = indexNames.containsKey(itemType) ? indexNames.get(itemType) :
(itemsMonthlyIndexed.contains(itemType) && dateHint != null ? getMonthlyIndex(dateHint) : indexName);
- client.prepareUpdate(index, itemType, itemId).setDoc(source)
+ client.prepareUpdate(index, itemType, itemId).setDoc(FieldDotEscaper.escapeMap(source))
.execute()
.actionGet();
return true;
- } catch (IndexMissingException e) {
+ } catch (IndexNotFoundException e) {
logger.debug("No index found for itemType=" + clazz.getName() + "itemId=" + itemId, e);
} catch (NoSuchFieldException e) {
logger.error("Error updating item " + itemId, e);
@@ -571,11 +617,12 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
String index = indexNames.containsKey(itemType) ? indexNames.get(itemType) :
(itemsMonthlyIndexed.contains(itemType) && dateHint != null ? getMonthlyIndex(dateHint) : indexName);
- client.prepareUpdate(index, itemType, itemId).setScript(script, ScriptService.ScriptType.INLINE).setScriptParams(scriptParams)
+ Script actualScript = new Script(script, ScriptService.ScriptType.INLINE, null, scriptParams);
+ client.prepareUpdate(index, itemType, itemId).setScript(actualScript)
.execute()
.actionGet();
return true;
- } catch (IndexMissingException e) {
+ } catch (IndexNotFoundException e) {
logger.debug("No index found for itemType=" + clazz.getName() + "itemId=" + itemId, e);
} catch (NoSuchFieldException e) {
logger.error("Error updating item " + itemId, e);
@@ -612,9 +659,12 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
try {
String itemType = (String) clazz.getField("ITEM_TYPE").get(null);
- client.prepareDeleteByQuery(getIndexNameForQuery(itemType))
+ DeleteByQueryResponse rsp = new DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE)
+ .setIndices(getIndexNameForQuery(itemType))
.setQuery(conditionESQueryBuilderDispatcher.getQueryBuilder(query))
- .execute().actionGet();
+ .execute()
+ .actionGet();
+
return true;
} catch (Exception e) {
logger.error("Cannot remove by query", e);
@@ -817,8 +867,8 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
final Class<? extends Item> clazz = item.getClass();
String itemType = (String) clazz.getField("ITEM_TYPE").get(null);
- FilterBuilder builder = FilterBuilders.andFilter(
- FilterBuilders.idsFilter(itemType).ids(item.getItemId()),
+ QueryBuilder builder = QueryBuilders.andQuery(
+ QueryBuilders.idsQuery(itemType).ids(item.getItemId()),
conditionESQueryBuilderDispatcher.buildFilter(query));
return queryCount(builder, itemType) > 0;
} catch (IllegalAccessException e) {
@@ -882,7 +932,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
return queryCount(conditionESQueryBuilderDispatcher.buildFilter(query), itemType);
}
- private long queryCount(final FilterBuilder filter, final String itemType) {
+ private long queryCount(final QueryBuilder filter, final String itemType) {
return new InClassLoaderExecute<Long>() {
@Override
@@ -960,7 +1010,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
results.add(value);
}
} catch (Exception t) {
- logger.error("Error loading itemType=" + clazz.getName() + "query=" + query, t);
+ logger.error("Error loading itemType=" + clazz.getName() + " query=" + query + " sortBy=" + sortBy, t);
}
return new PartialList<T>(results, offset, size, totalHits);
@@ -987,7 +1037,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
AggregationBuilder bucketsAggregation = null;
if (aggregate instanceof DateAggregate) {
DateAggregate dateAggregate = (DateAggregate) aggregate;
- DateHistogramBuilder dateHistogramBuilder = AggregationBuilders.dateHistogram("buckets").field(aggregate.getField()).interval(new DateHistogram.Interval((dateAggregate.getInterval())));
+ DateHistogramBuilder dateHistogramBuilder = AggregationBuilders.dateHistogram("buckets").field(aggregate.getField()).interval(new DateHistogramInterval((dateAggregate.getInterval())));
if (dateAggregate.getFormat() != null) {
dateHistogramBuilder.format(dateAggregate.getFormat());
}
@@ -1073,7 +1123,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
if (aggregations.get("buckets") != null) {
MultiBucketsAggregation terms = aggregations.get("buckets");
for (MultiBucketsAggregation.Bucket bucket : terms.getBuckets()) {
- results.put(bucket.getKey(), bucket.getDocCount());
+ results.put(bucket.getKeyAsString(), bucket.getDocCount());
}
SingleBucketAggregation missing = aggregations.get("missing");
if (missing.getDocCount() > 0) {
@@ -1116,7 +1166,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
protected List<ClusterNode> execute(Object... args) {
Map<String, ClusterNode> clusterNodes = new LinkedHashMap<String, ClusterNode>();
- NodesInfoResponse nodesInfoResponse = client.admin().cluster().prepareNodesInfo(NodesOperationRequest.ALL_NODES)
+ NodesInfoResponse nodesInfoResponse = client.admin().cluster().prepareNodesInfo(NodesInfoRequest.ALL_NODES)
.setSettings(true)
.execute()
.actionGet();
@@ -1135,8 +1185,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
}
- NodesStatsResponse nodesStatsResponse = client.admin().cluster().prepareNodesStats(NodesOperationRequest.ALL_NODES)
+ NodesStatsResponse nodesStatsResponse = client.admin().cluster().prepareNodesStats(NodesInfoRequest.ALL_NODES)
.setOs(true)
+ .setJvm(true)
.setProcess(true)
.execute()
.actionGet();
@@ -1152,8 +1203,8 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
clusterNode.setCpuLoad(nodeStats.getProcess().getCpu().getPercent());
}
if (nodeStats.getOs() != null) {
- clusterNode.setLoadAverage(nodeStats.getOs().getLoadAverage());
- clusterNode.setUptime(nodeStats.getOs().getUptime().getMillis());
+ clusterNode.setLoadAverage(new double[] { nodeStats.getOs().getLoadAverage() });
+ clusterNode.setUptime(nodeStats.getJvm().getUptime().getMillis());
}
}
}
@@ -1335,5 +1386,11 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
}
-
+ private String getConfig(Map<String,String> settings, String key,
+ String defaultValue) {
+ if (settings != null && settings.get(key) != null) {
+ return settings.get(key);
+ }
+ return defaultValue;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscaper.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscaper.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscaper.java
new file mode 100644
index 0000000..62ccacc
--- /dev/null
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscaper.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.persistence.elasticsearch;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+/**
+ * A utility class to un/escape field names that contain dots that are not allowed in ElasticSearch 2.x
+ */
+public class FieldDotEscaper {
+
+ public static final String DOT_ESCAPE_MARKER = "__DOT__";
+
+ public static String escapeJson(final String jsonInput) {
+ return escapeJson(jsonInput, null);
+ }
+
+ public static String escapeJson(final String jsonInput, final Set<String> modifiedNames) {
+ if (!jsonInput.contains(".")) { // optimization in case no dot is present at all
+ return jsonInput;
+ }
+ StringBuffer result = new StringBuffer();
+ FieldDotJsonTransformer jsonTransformer = new FieldDotJsonTransformer(jsonInput, result, DOT_ESCAPE_MARKER);
+ Set<String> pathsModified = jsonTransformer.transform();
+ if (modifiedNames != null) {
+ modifiedNames.addAll(pathsModified);
+ }
+ return result.toString();
+ }
+
+ public static String unescapeJson(final String jsonInput) {
+ return unescapeString(jsonInput);
+ }
+
+ public static String escapeString(final String stringInput) {
+ return stringInput.replaceAll("\\.", DOT_ESCAPE_MARKER);
+ }
+
+ public static String unescapeString(final String stringInput) {
+ return stringInput.replaceAll(DOT_ESCAPE_MARKER, ".");
+ }
+
+ public static Map<? extends String, ?> escapeMap(final Map<? extends String, ?> mapInput) {
+ Map<String,Object> result = new LinkedHashMap<>(mapInput.size());
+ for (Map.Entry<? extends String, ? extends Object> entry : mapInput.entrySet()) {
+ String entryKey = entry.getKey();
+ if (entryKey.contains(".")) {
+ entryKey = escapeString(entryKey);
+ }
+ result.put(entryKey, entry.getValue());
+ }
+ return result;
+ }
+
+ public static Map<? extends String, ?> unescapeMap(final Map<? extends String, ?> mapInput) {
+ Map<String, Object> result = new LinkedHashMap<>(mapInput.size());
+ for (Map.Entry<? extends String, ?> entry : mapInput.entrySet()) {
+ String entryKey = entry.getKey();
+ if (entryKey.contains(DOT_ESCAPE_MARKER)) {
+ entryKey = unescapeString(entryKey);
+ }
+ result.put(entryKey, entry.getValue());
+ }
+ return result;
+ }
+
+ public static Properties escapeProperties(final Properties input) {
+ Properties result = new Properties();
+ for (String propertyName : input.stringPropertyNames()) {
+ String newPropertyName = propertyName;
+ if (propertyName.contains(".")) {
+ newPropertyName = escapeString(propertyName);
+ }
+ result.put(newPropertyName, input.getProperty(propertyName));
+ }
+ return result;
+ }
+
+ public static Properties unescapeProperties(final Properties input) {
+ Properties result = new Properties();
+ for (String propertyName : input.stringPropertyNames()) {
+ String newPropertyName = propertyName;
+ if (propertyName.contains(DOT_ESCAPE_MARKER)) {
+ newPropertyName = unescapeString(propertyName);
+ }
+ result.put(newPropertyName, input.getProperty(propertyName));
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformer.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformer.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformer.java
new file mode 100644
index 0000000..a26e3f4
--- /dev/null
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformer.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.persistence.elasticsearch;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Deque;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.Set;
+
+/**
+ * A custom JSON transformer that can replace dot characters in field names with a marker. This is useful for tools like
+ * ElasticSearch 2.x that doesn't allow dot characters in field names since version 2.x
+ */
+public class FieldDotJsonTransformer {
+
+ public static final char BEGIN_ARRAY_CHAR = '[';
+ public static final char END_ARRAY_CHAR = ']';
+ public static final char BEGIN_OBJECT_CHAR = '{';
+ public static final char END_OBJECT_CHAR = '}';
+ public static final char NAME_SEPARATOR_CHAR = ':';
+ public static final char VALUE_SEPARATOR_CHAR = ',';
+
+ public static final char STRING_BEGIN_OR_END_CHAR = '"';
+ public static final char STRING_ESCAPE_CHAR = '\\';
+ public static final char STRING_UNICODE_CHAR = 'u';
+ public static final char STRING_DOT_CHAR = '.';
+
+ public static final String WHITESPACE_CHARS = " \t\n\r";
+ public static final String NUMBER_CHARS = "+-0123456789eE.";
+
+ String jsonInput;
+ int pos = -1;
+ StringBuffer output;
+ String dotReplacement = null;
+ Set<String> modifiedNames = new LinkedHashSet<>();
+ Deque<String> currentPath = new LinkedList<>();
+
+ public FieldDotJsonTransformer(String jsonInput, StringBuffer output, String dotReplacement) {
+ this.jsonInput = jsonInput;
+ this.output = output;
+ this.dotReplacement = dotReplacement;
+ }
+
+ public Set<String> transform() {
+ parseValue();
+ return modifiedNames;
+ }
+
+ protected Character getNextChar() {
+ pos++;
+ char ch = jsonInput.charAt(pos);
+ if (pos >= jsonInput.length()) {
+ return null;
+ }
+ return ch;
+ }
+
+ protected Character peekNextToken() {
+ if (pos + 1 >= jsonInput.length()) {
+ return null;
+ }
+ int i = 1;
+ Character ch = jsonInput.charAt(pos + i);
+ while (WHITESPACE_CHARS.indexOf(ch) > -1 && (pos + i < jsonInput.length())) {
+ i++;
+ ch = jsonInput.charAt(pos + i);
+ }
+ return ch;
+ }
+
+ protected Character getNextToken() {
+ Character ch = getNextChar();
+ while ((ch != null) && (WHITESPACE_CHARS.indexOf(ch) > -1)) {
+ output.append(ch);
+ ch = getNextChar();
+ }
+ return ch;
+ }
+
+ protected void parseBooleanValue(boolean expectedValue) {
+ if (expectedValue) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getNextToken());
+ sb.append(getNextChar());
+ sb.append(getNextChar());
+ sb.append(getNextChar());
+ if ("true".equals(sb.toString())) {
+ // everything matches
+ }
+ output.append(sb.toString());
+ } else {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getNextToken());
+ sb.append(getNextChar());
+ sb.append(getNextChar());
+ sb.append(getNextChar());
+ sb.append(getNextChar());
+ if ("false".equals(sb.toString())) {
+ // everything matches
+ }
+ output.append(sb.toString());
+ }
+ }
+
+ protected void parseNullValue() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getNextToken());
+ sb.append(getNextChar());
+ sb.append(getNextChar());
+ sb.append(getNextChar());
+ if ("null".equals(sb.toString())) {
+ // everything matches
+ }
+ output.append(sb.toString());
+ }
+
+ protected String parseString(boolean escapeDots) {
+ Character ch = getNextToken();
+ if (ch != STRING_BEGIN_OR_END_CHAR) {
+ return null;
+ }
+ output.append(ch);
+ boolean modified = false;
+ StringBuilder stringContent = new StringBuilder();
+ while ((ch = getNextChar()) != null) {
+ switch (ch) {
+ case STRING_ESCAPE_CHAR:
+ stringContent.append(ch);
+ output.append(ch);
+ ch = getNextChar();
+ if (ch == STRING_UNICODE_CHAR) {
+ // case of Unicode escape sequence
+ }
+ output.append(ch);
+ stringContent.append(ch);
+ break;
+ case STRING_DOT_CHAR:
+ if (escapeDots && dotReplacement != null) {
+ output.append(dotReplacement);
+ modified = true;
+ } else {
+ output.append(ch);
+ }
+ stringContent.append(ch);
+ break;
+ case STRING_BEGIN_OR_END_CHAR:
+ output.append(ch);
+ if (modified) {
+ modifiedNames.add(StringUtils.join(currentPath, "/") + "/" + stringContent.toString());
+ }
+ return stringContent.toString();
+ default:
+ output.append(ch);
+ stringContent.append(ch);
+ }
+ }
+ return null;
+ }
+
+ protected void parseNumber() {
+ StringBuilder sb = new StringBuilder();
+ Character ch = peekNextToken();
+ while ((ch != null) && (NUMBER_CHARS.indexOf(ch) > -1)) {
+ ch = getNextChar();
+ sb.append(ch);
+ ch = peekNextToken();
+ }
+ output.append(sb.toString());
+ }
+
+ protected void parseValue() {
+ char ch = peekNextToken();
+ // we've got to identify the type or value first
+ switch (ch) {
+ case 't': // true
+ parseBooleanValue(true);
+ break;
+ case 'f': // false
+ parseBooleanValue(false);
+ break;
+ case 'n': // null
+ parseNullValue();
+ break;
+ case BEGIN_OBJECT_CHAR:
+ parseObject();
+ break;
+ case BEGIN_ARRAY_CHAR:
+ parseArray();
+ break;
+ case STRING_BEGIN_OR_END_CHAR:
+ parseString(false);
+ break;
+ default:
+ parseNumber();
+ }
+ }
+
+ protected void parseObject() {
+ Character ch = getNextToken();
+ if (ch != BEGIN_OBJECT_CHAR) {
+ return;
+ }
+ output.append(ch);
+ // now let's check the case of an empty object
+ ch = peekNextToken();
+ if (ch == END_OBJECT_CHAR) {
+ ch = getNextToken();
+ output.append(ch);
+ return;
+ }
+ if (parseNameValuePair()) {
+ return;
+ }
+ while ((ch = getNextToken()) != null) {
+ output.append(ch);
+ switch (ch) {
+ case VALUE_SEPARATOR_CHAR:
+ parseNameValuePair();
+ break;
+ case END_OBJECT_CHAR:
+ return;
+ default:
+ return;
+ }
+ }
+ }
+
+ protected void parseArray() {
+ Character ch = getNextToken();
+ if (ch != BEGIN_ARRAY_CHAR) {
+ return;
+ }
+ output.append(ch);
+ // now let's check the case of an empty array
+ ch = peekNextToken();
+ if (ch == END_ARRAY_CHAR) {
+ ch = getNextToken();
+ output.append(ch);
+ return;
+ }
+ parseValue();
+ while ((ch = getNextToken()) != null) {
+ output.append(ch);
+ switch (ch) {
+ case VALUE_SEPARATOR_CHAR:
+ parseValue();
+ break;
+ case END_ARRAY_CHAR:
+ return;
+ default:
+ return;
+ }
+ }
+ }
+
+ protected boolean parseNameValuePair() {
+ Character ch;
+ String name = parseString(true);
+ if (name != null) {
+ currentPath.addLast(name);
+ }
+ ch = getNextToken();
+ if (ch != NAME_SEPARATOR_CHAR) {
+ return true;
+ }
+ output.append(ch);
+ parseValue();
+ if (name != null) {
+ currentPath.removeLast();
+ }
+ return false;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java
index 3666d8d..63964e1 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java
@@ -17,12 +17,12 @@
package org.apache.unomi.persistence.elasticsearch.conditions;
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
import org.apache.lucene.util.ArrayUtil;
import org.apache.unomi.api.conditions.Condition;
-import org.elasticsearch.common.base.Function;
-import org.elasticsearch.common.collect.Lists;
import org.mvel2.MVEL;
import org.mvel2.ParserConfiguration;
import org.mvel2.ParserContext;
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilder.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilder.java
index cc2dc89..78a5cba 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilder.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilder.java
@@ -18,12 +18,12 @@
package org.apache.unomi.persistence.elasticsearch.conditions;
import org.apache.unomi.api.conditions.Condition;
-import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
import java.util.Map;
public interface ConditionESQueryBuilder {
- FilterBuilder buildFilter(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher);
+ QueryBuilder buildQuery(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher);
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java
index 526b49f..5289f69 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java
@@ -18,9 +18,7 @@
package org.apache.unomi.persistence.elasticsearch.conditions;
import org.apache.unomi.api.conditions.Condition;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
-import org.elasticsearch.index.query.FilteredQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.osgi.framework.BundleContext;
import org.slf4j.Logger;
@@ -67,15 +65,15 @@ public class ConditionESQueryBuilderDispatcher {
return "{\"query\": " + getQueryBuilder(condition).toString() + "}";
}
- public FilteredQueryBuilder getQueryBuilder(Condition condition) {
- return QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), buildFilter(condition));
+ public QueryBuilder getQueryBuilder(Condition condition) {
+ return QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()).filter(buildFilter(condition));
}
- public FilterBuilder buildFilter(Condition condition) {
+ public QueryBuilder buildFilter(Condition condition) {
return buildFilter(condition, new HashMap<String, Object>());
}
- public FilterBuilder buildFilter(Condition condition, Map<String, Object> context) {
+ public QueryBuilder buildFilter(Condition condition, Map<String, Object> context) {
if(condition == null || condition.getConditionType() == null) {
throw new IllegalArgumentException("Condition is null or doesn't have type, impossible to build filter");
}
@@ -94,12 +92,12 @@ public class ConditionESQueryBuilderDispatcher {
ConditionESQueryBuilder queryBuilder = queryBuilders.get(queryBuilderKey);
Condition contextualCondition = ConditionContextHelper.getContextualCondition(condition, context);
if (contextualCondition != null) {
- return queryBuilder.buildFilter(contextualCondition, context, this);
+ return queryBuilder.buildQuery(contextualCondition, context, this);
}
}
// if no matching
- return FilterBuilders.matchAllFilter();
+ return QueryBuilders.matchAllQuery();
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
index c469be8..36788d7 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
@@ -22,11 +22,11 @@
},
"startDate": {
"type": "date",
- "format": "dateOptionalTime"
+ "format": "strict_date_optional_time||epoch_millis"
},
"endDate": {
"type": "date",
- "format": "dateOptionalTime"
+ "format": "strict_date_optional_time||epoch_millis"
},
"itemId": {
"type": "string",
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
index e9ce4d3..ccc001e 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
@@ -26,7 +26,7 @@
},
"eventDate": {
"type": "date",
- "format": "dateOptionalTime"
+ "format": "strict_date_optional_time||epoch_millis"
},
"itemId": {
"type": "string",
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
index abe1dbc..6089499 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
@@ -39,7 +39,7 @@
},
"timeStamp": {
"type": "date",
- "format": "dateOptionalTime"
+ "format": "strict_date_optional_time||epoch_millis"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
index bfd6939..4a20426 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
@@ -31,7 +31,7 @@
"properties": {
"firstVisit": {
"type": "date",
- "format": "dateOptionalTime"
+ "format": "strict_date_optional_time||epoch_millis"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
index ef808ed..2047b4e 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
@@ -26,7 +26,7 @@
},
"lastEventDate": {
"type": "date",
- "format": "dateOptionalTime"
+ "format": "strict_date_optional_time||epoch_millis"
},
"profileId": {
"type": "string",
@@ -51,7 +51,7 @@
},
"timeStamp": {
"type": "date",
- "format": "dateOptionalTime"
+ "format": "strict_date_optional_time||epoch_millis"
}
}
}