You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by sh...@apache.org on 2016/09/22 14:03:26 UTC
incubator-unomi git commit: Use ElasticSearch's re-introduced support
for dots in names and remove the old dot escaping code.
Repository: incubator-unomi
Updated Branches:
refs/heads/UNOMI-28-ES-2-X-UPGRADE 12641b9b3 -> fda0d9ce7
Use ElasticSearch's re-introduced support for dots in names and remove the old dot escaping code.
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/fda0d9ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/fda0d9ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/fda0d9ce
Branch: refs/heads/UNOMI-28-ES-2-X-UPGRADE
Commit: fda0d9ce711ff189a8f4af42711d17a5d9d020ed
Parents: 12641b9
Author: Serge Huber <sh...@apache.org>
Authored: Thu Sep 22 16:03:18 2016 +0200
Committer: Serge Huber <sh...@apache.org>
Committed: Thu Sep 22 16:03:18 2016 +0200
----------------------------------------------------------------------
.../ElasticSearchPersistenceServiceImpl.java | 13 +-
.../elasticsearch/FieldDotEscaper.java | 108 -------
.../elasticsearch/FieldDotJsonTransformer.java | 290 -------------------
.../elasticsearch/FieldDotEscapeTest.java | 95 ------
.../FieldDotJsonTransformerTest.java | 61 ----
.../core/src/test/resources/complex.json | 50 ----
.../core/src/test/resources/profile.json | 1 -
7 files changed, 6 insertions(+), 612 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/fda0d9ce/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index b5f1c7a..16f1bc6 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -132,6 +132,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
public static final String PATH_HOME = "path.home";
public static final String PATH_PLUGINS = "path.plugins";
public static final String INDEX_MAX_RESULT_WINDOW = "index.max_result_window";
+ public static final String MAPPER_ALLOW_DOTS_IN_NAME = "mapper.allow_dots_in_name";
private Node node;
private Client client;
@@ -281,6 +282,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
File homeFile = new File(getConfig(settings, PATH_HOME, new File(new File(karafHome), ELASTICSEARCH_HOME_DIRECTORY).getAbsolutePath()));
File dataFile = new File(getConfig(settings, PATH_DATA, new File(new File(karafHome), ELASTICSEARCH_DATA_DIRECTORY).getAbsolutePath()));
+ // allow dots in mappings (re-introduced in ElasticSearch 2.4.0)
+ System.setProperty(MAPPER_ALLOW_DOTS_IN_NAME, "true");
+
settingsBuilder.put(CLUSTER_NAME, clusterName)
.put(NODE_DATA, nodeData)
.put(PATH_DATA, dataFile.getAbsolutePath())
@@ -528,7 +532,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
.actionGet();
if (response.isExists()) {
String sourceAsString = response.getSourceAsString();
- final T value = CustomObjectMapper.getObjectMapper().readValue(FieldDotEscaper.unescapeJson(sourceAsString), clazz);
+ final T value = CustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz);
value.setItemId(response.getId());
return value;
} else {
@@ -555,11 +559,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
protected Boolean execute(Object... args) {
try {
String source = CustomObjectMapper.getObjectMapper().writeValueAsString(item);
- Set<String> modifiedNames = new LinkedHashSet<>();
- source = FieldDotEscaper.escapeJson(source, modifiedNames);
- if (modifiedNames.size() > 0) {
- logger.warn("Found JSON property names with dot characters not allowed by ElasticSearch 2.x={} in item {}", modifiedNames, item);
- }
String itemType = item.getItemType();
String index = indexNames.containsKey(itemType) ? indexNames.get(itemType) :
(itemsMonthlyIndexed.contains(itemType) ? getMonthlyIndex(((TimestampedItem) item).getTimeStamp()) : indexName);
@@ -606,7 +605,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
String index = indexNames.containsKey(itemType) ? indexNames.get(itemType) :
(itemsMonthlyIndexed.contains(itemType) && dateHint != null ? getMonthlyIndex(dateHint) : indexName);
- client.prepareUpdate(index, itemType, itemId).setDoc(FieldDotEscaper.escapeMap(source))
+ client.prepareUpdate(index, itemType, itemId).setDoc(source)
.execute()
.actionGet();
return true;
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/fda0d9ce/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscaper.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscaper.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscaper.java
deleted file mode 100644
index 62ccacc..0000000
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscaper.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.unomi.persistence.elasticsearch;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-
-/**
- * A utility class to un/escape field names that contain dots that are not allowed in ElasticSearch 2.x
- */
-public class FieldDotEscaper {
-
- public static final String DOT_ESCAPE_MARKER = "__DOT__";
-
- public static String escapeJson(final String jsonInput) {
- return escapeJson(jsonInput, null);
- }
-
- public static String escapeJson(final String jsonInput, final Set<String> modifiedNames) {
- if (!jsonInput.contains(".")) { // optimization in case no dot is present at all
- return jsonInput;
- }
- StringBuffer result = new StringBuffer();
- FieldDotJsonTransformer jsonTransformer = new FieldDotJsonTransformer(jsonInput, result, DOT_ESCAPE_MARKER);
- Set<String> pathsModified = jsonTransformer.transform();
- if (modifiedNames != null) {
- modifiedNames.addAll(pathsModified);
- }
- return result.toString();
- }
-
- public static String unescapeJson(final String jsonInput) {
- return unescapeString(jsonInput);
- }
-
- public static String escapeString(final String stringInput) {
- return stringInput.replaceAll("\\.", DOT_ESCAPE_MARKER);
- }
-
- public static String unescapeString(final String stringInput) {
- return stringInput.replaceAll(DOT_ESCAPE_MARKER, ".");
- }
-
- public static Map<? extends String, ?> escapeMap(final Map<? extends String, ?> mapInput) {
- Map<String,Object> result = new LinkedHashMap<>(mapInput.size());
- for (Map.Entry<? extends String, ? extends Object> entry : mapInput.entrySet()) {
- String entryKey = entry.getKey();
- if (entryKey.contains(".")) {
- entryKey = escapeString(entryKey);
- }
- result.put(entryKey, entry.getValue());
- }
- return result;
- }
-
- public static Map<? extends String, ?> unescapeMap(final Map<? extends String, ?> mapInput) {
- Map<String, Object> result = new LinkedHashMap<>(mapInput.size());
- for (Map.Entry<? extends String, ?> entry : mapInput.entrySet()) {
- String entryKey = entry.getKey();
- if (entryKey.contains(DOT_ESCAPE_MARKER)) {
- entryKey = unescapeString(entryKey);
- }
- result.put(entryKey, entry.getValue());
- }
- return result;
- }
-
- public static Properties escapeProperties(final Properties input) {
- Properties result = new Properties();
- for (String propertyName : input.stringPropertyNames()) {
- String newPropertyName = propertyName;
- if (propertyName.contains(".")) {
- newPropertyName = escapeString(propertyName);
- }
- result.put(newPropertyName, input.getProperty(propertyName));
- }
- return result;
- }
-
- public static Properties unescapeProperties(final Properties input) {
- Properties result = new Properties();
- for (String propertyName : input.stringPropertyNames()) {
- String newPropertyName = propertyName;
- if (propertyName.contains(DOT_ESCAPE_MARKER)) {
- newPropertyName = unescapeString(propertyName);
- }
- result.put(newPropertyName, input.getProperty(propertyName));
- }
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/fda0d9ce/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformer.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformer.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformer.java
deleted file mode 100644
index a26e3f4..0000000
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformer.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.unomi.persistence.elasticsearch;
-
-import org.apache.commons.lang3.StringUtils;
-
-import java.util.Deque;
-import java.util.LinkedHashSet;
-import java.util.LinkedList;
-import java.util.Set;
-
-/**
- * A custom JSON transformer that can replace dot characters in field names with a marker. This is useful for tools like
- * ElasticSearch 2.x that doesn't allow dot characters in field names since version 2.x
- */
-public class FieldDotJsonTransformer {
-
- public static final char BEGIN_ARRAY_CHAR = '[';
- public static final char END_ARRAY_CHAR = ']';
- public static final char BEGIN_OBJECT_CHAR = '{';
- public static final char END_OBJECT_CHAR = '}';
- public static final char NAME_SEPARATOR_CHAR = ':';
- public static final char VALUE_SEPARATOR_CHAR = ',';
-
- public static final char STRING_BEGIN_OR_END_CHAR = '"';
- public static final char STRING_ESCAPE_CHAR = '\\';
- public static final char STRING_UNICODE_CHAR = 'u';
- public static final char STRING_DOT_CHAR = '.';
-
- public static final String WHITESPACE_CHARS = " \t\n\r";
- public static final String NUMBER_CHARS = "+-0123456789eE.";
-
- String jsonInput;
- int pos = -1;
- StringBuffer output;
- String dotReplacement = null;
- Set<String> modifiedNames = new LinkedHashSet<>();
- Deque<String> currentPath = new LinkedList<>();
-
- public FieldDotJsonTransformer(String jsonInput, StringBuffer output, String dotReplacement) {
- this.jsonInput = jsonInput;
- this.output = output;
- this.dotReplacement = dotReplacement;
- }
-
- public Set<String> transform() {
- parseValue();
- return modifiedNames;
- }
-
- protected Character getNextChar() {
- pos++;
- char ch = jsonInput.charAt(pos);
- if (pos >= jsonInput.length()) {
- return null;
- }
- return ch;
- }
-
- protected Character peekNextToken() {
- if (pos + 1 >= jsonInput.length()) {
- return null;
- }
- int i = 1;
- Character ch = jsonInput.charAt(pos + i);
- while (WHITESPACE_CHARS.indexOf(ch) > -1 && (pos + i < jsonInput.length())) {
- i++;
- ch = jsonInput.charAt(pos + i);
- }
- return ch;
- }
-
- protected Character getNextToken() {
- Character ch = getNextChar();
- while ((ch != null) && (WHITESPACE_CHARS.indexOf(ch) > -1)) {
- output.append(ch);
- ch = getNextChar();
- }
- return ch;
- }
-
- protected void parseBooleanValue(boolean expectedValue) {
- if (expectedValue) {
- StringBuilder sb = new StringBuilder();
- sb.append(getNextToken());
- sb.append(getNextChar());
- sb.append(getNextChar());
- sb.append(getNextChar());
- if ("true".equals(sb.toString())) {
- // everything matches
- }
- output.append(sb.toString());
- } else {
- StringBuilder sb = new StringBuilder();
- sb.append(getNextToken());
- sb.append(getNextChar());
- sb.append(getNextChar());
- sb.append(getNextChar());
- sb.append(getNextChar());
- if ("false".equals(sb.toString())) {
- // everything matches
- }
- output.append(sb.toString());
- }
- }
-
- protected void parseNullValue() {
- StringBuilder sb = new StringBuilder();
- sb.append(getNextToken());
- sb.append(getNextChar());
- sb.append(getNextChar());
- sb.append(getNextChar());
- if ("null".equals(sb.toString())) {
- // everything matches
- }
- output.append(sb.toString());
- }
-
- protected String parseString(boolean escapeDots) {
- Character ch = getNextToken();
- if (ch != STRING_BEGIN_OR_END_CHAR) {
- return null;
- }
- output.append(ch);
- boolean modified = false;
- StringBuilder stringContent = new StringBuilder();
- while ((ch = getNextChar()) != null) {
- switch (ch) {
- case STRING_ESCAPE_CHAR:
- stringContent.append(ch);
- output.append(ch);
- ch = getNextChar();
- if (ch == STRING_UNICODE_CHAR) {
- // case of Unicode escape sequence
- }
- output.append(ch);
- stringContent.append(ch);
- break;
- case STRING_DOT_CHAR:
- if (escapeDots && dotReplacement != null) {
- output.append(dotReplacement);
- modified = true;
- } else {
- output.append(ch);
- }
- stringContent.append(ch);
- break;
- case STRING_BEGIN_OR_END_CHAR:
- output.append(ch);
- if (modified) {
- modifiedNames.add(StringUtils.join(currentPath, "/") + "/" + stringContent.toString());
- }
- return stringContent.toString();
- default:
- output.append(ch);
- stringContent.append(ch);
- }
- }
- return null;
- }
-
- protected void parseNumber() {
- StringBuilder sb = new StringBuilder();
- Character ch = peekNextToken();
- while ((ch != null) && (NUMBER_CHARS.indexOf(ch) > -1)) {
- ch = getNextChar();
- sb.append(ch);
- ch = peekNextToken();
- }
- output.append(sb.toString());
- }
-
- protected void parseValue() {
- char ch = peekNextToken();
- // we've got to identify the type or value first
- switch (ch) {
- case 't': // true
- parseBooleanValue(true);
- break;
- case 'f': // false
- parseBooleanValue(false);
- break;
- case 'n': // null
- parseNullValue();
- break;
- case BEGIN_OBJECT_CHAR:
- parseObject();
- break;
- case BEGIN_ARRAY_CHAR:
- parseArray();
- break;
- case STRING_BEGIN_OR_END_CHAR:
- parseString(false);
- break;
- default:
- parseNumber();
- }
- }
-
- protected void parseObject() {
- Character ch = getNextToken();
- if (ch != BEGIN_OBJECT_CHAR) {
- return;
- }
- output.append(ch);
- // now let's check the case of an empty object
- ch = peekNextToken();
- if (ch == END_OBJECT_CHAR) {
- ch = getNextToken();
- output.append(ch);
- return;
- }
- if (parseNameValuePair()) {
- return;
- }
- while ((ch = getNextToken()) != null) {
- output.append(ch);
- switch (ch) {
- case VALUE_SEPARATOR_CHAR:
- parseNameValuePair();
- break;
- case END_OBJECT_CHAR:
- return;
- default:
- return;
- }
- }
- }
-
- protected void parseArray() {
- Character ch = getNextToken();
- if (ch != BEGIN_ARRAY_CHAR) {
- return;
- }
- output.append(ch);
- // now let's check the case of an empty array
- ch = peekNextToken();
- if (ch == END_ARRAY_CHAR) {
- ch = getNextToken();
- output.append(ch);
- return;
- }
- parseValue();
- while ((ch = getNextToken()) != null) {
- output.append(ch);
- switch (ch) {
- case VALUE_SEPARATOR_CHAR:
- parseValue();
- break;
- case END_ARRAY_CHAR:
- return;
- default:
- return;
- }
- }
- }
-
- protected boolean parseNameValuePair() {
- Character ch;
- String name = parseString(true);
- if (name != null) {
- currentPath.addLast(name);
- }
- ch = getNextToken();
- if (ch != NAME_SEPARATOR_CHAR) {
- return true;
- }
- output.append(ch);
- parseValue();
- if (name != null) {
- currentPath.removeLast();
- }
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/fda0d9ce/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscapeTest.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscapeTest.java b/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscapeTest.java
deleted file mode 100644
index c4322dc..0000000
--- a/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscapeTest.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.unomi.persistence.elasticsearch;
-
-import org.apache.commons.io.IOUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.*;
-
-/**
- * A unit test class for the FieldDotEscaper utility methods
- */
-public class FieldDotEscapeTest {
-
- @Test
- public void testTrickyJson() {
- String result = FieldDotEscaper.escapeJson("{\"tricking_the_parser\" : \"this.should.not\\\": match.either\"}");
- Assert.assertTrue("Found escaped pattern instead of untouched one", result.contains("this.should.not\\\": match.either"));
- }
-
- @Test
- public void testJustDot() {
- String result = FieldDotEscaper.escapeJson("{\".\" : \"this.should.not\\\": match.either\"}");
- Assert.assertTrue("Didn't find expected escaped pattern", result.contains("\"__DOT__\""));
- Assert.assertTrue("Found escaped pattern instead of untouched one", result.contains("this.should.not\\\": match.either"));
- }
-
- @Test
- public void testComplexJson() throws IOException {
- InputStream complexJsonInputStream = this.getClass().getClassLoader().getResourceAsStream("complex.json");
- String input = IOUtils.toString(complexJsonInputStream);
- System.out.println("Original JSON:");
- System.out.println("=================");
- System.out.println(input);
- Set<String> modifiedNames = new LinkedHashSet<>();
- String result = FieldDotEscaper.escapeJson(input, modifiedNames);
- System.out.println("Modified names:");
- System.out.println("=================");
- for (String modifiedName : modifiedNames) {
- System.out.println(modifiedName);
- }
- System.out.println("Transformed JSON:");
- System.out.println("=================");
- System.out.println(result);
- Assert.assertTrue("Didn't find expected escaped pattern", result.contains("src_terms[0]__DOT__fields__DOT__siteContent"));
- Assert.assertTrue("Didn't find expected escaped pattern", result.contains("newline__DOT__test"));
- Assert.assertTrue("Found escaped pattern instead of untouched one", result.contains("this.should.never:match"));
- Assert.assertTrue("Found escaped pattern instead of untouched one", result.contains("this.should.not\\\": match.either"));
- result = FieldDotEscaper.unescapeJson(result);
- Assert.assertEquals("Round trip of escaping then unescaping should be identical", input, result);
- }
-
- @Test
- public void testString() {
- String input = "this.is.a..test";
- String result = FieldDotEscaper.unescapeString(FieldDotEscaper.escapeString(input));
- Assert.assertEquals("Strings should be exactly the same", input, result);
- }
-
- @Test
- public void testMap() {
- Map<String,Object> input = new HashMap<>();
- input.put("this.is.a..test", "1");
- input.put("another.test", "2");
- Map<? extends String,?> result = FieldDotEscaper.unescapeMap(FieldDotEscaper.escapeMap(input));
- Assert.assertEquals("Maps should be identical", input, result);
- }
-
- @Test
- public void testProperties() {
- Properties input = new Properties();
- input.put("this.is.a..test", "1");
- input.put("another.test", "2");
- Properties result = FieldDotEscaper.unescapeProperties(FieldDotEscaper.escapeProperties(input));
- Assert.assertEquals("Properties should be identical", input, result);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/fda0d9ce/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformerTest.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformerTest.java b/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformerTest.java
deleted file mode 100644
index 284ac84..0000000
--- a/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformerTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.unomi.persistence.elasticsearch;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.io.IOUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Map;
-
-/**
- * A unit test class for our custom JSON parser.
- */
-public class FieldDotJsonTransformerTest {
-
- @Test
- public void testNoReplacement() throws IOException {
- InputStream complexJsonInputStream = this.getClass().getClassLoader().getResourceAsStream("complex.json");
- String input = IOUtils.toString(complexJsonInputStream);
- StringBuffer output = new StringBuffer();
- FieldDotJsonTransformer fieldDotJsonTransformer = new FieldDotJsonTransformer(input, output, null);
- fieldDotJsonTransformer.transform();
- Assert.assertEquals("Parsed JSON and original should be identical", input, output.toString());
- InputStream profileJsonInputStream = this.getClass().getClassLoader().getResourceAsStream("profile.json");
- input = IOUtils.toString(profileJsonInputStream);
- output = new StringBuffer();
- fieldDotJsonTransformer = new FieldDotJsonTransformer(input, output, null);
- fieldDotJsonTransformer.transform();
- Assert.assertEquals("Parsed JSON and original should be identical", input, output.toString());
- }
-
- @Test
- public void testWithJackson() throws IOException {
- InputStream complexJsonInputStream = this.getClass().getClassLoader().getResourceAsStream("complex.json");
- String input = IOUtils.toString(complexJsonInputStream);
- StringBuffer output = new StringBuffer();
- FieldDotJsonTransformer fieldDotJsonTransformer = new FieldDotJsonTransformer(input, output, FieldDotEscaper.DOT_ESCAPE_MARKER);
- fieldDotJsonTransformer.transform();
- ObjectMapper objectMapper = new ObjectMapper();
- Map<String,Object> object = objectMapper.readValue(output.toString(), Map.class);
- Assert.assertNotNull("Object parsed with Jackson should not be null", object);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/fda0d9ce/persistence-elasticsearch/core/src/test/resources/complex.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/test/resources/complex.json b/persistence-elasticsearch/core/src/test/resources/complex.json
deleted file mode 100644
index c6ff41c..0000000
--- a/persistence-elasticsearch/core/src/test/resources/complex.json
+++ /dev/null
@@ -1,50 +0,0 @@
-{
- "events": [
- {
- "scope": "ACMESPACE",
- "eventType": "form",
- "source": {
- "itemType": "page",
- "scope": "ACMESPACE",
- "itemId": "${pageId}",
- "properties": {
- "pageInfo": {
- "pageID": "${pageId}",
- "pageName": "${pageName}",
- "pagePath": "${pagePath}",
- "destinationURL": "${destinationURL}",
- "referringURL": "${previousURL}",
- "language": "${language}"
- },
- "category": {},
- "attributes": {
- "colon_in_value" : "this.should.never:match",
- "newline.test"
- : "more.dots",
- "tricking_the_parser" : "this.should.not\": match.either",
- "unicode black star": "before\u2605after",
- "escaping test" : "\"\n\r\t\\"
- }
- }
- },
- "target": {
- "itemType": "form",
- "scope": "ACMESPACE",
- "itemId": "searchForm"
- },
- "properties": {
- "jcrMethodToCall": "get",
- "src_originSiteKey": "ACMESPACE",
- "src_terms[0].term": "${word}",
- "src_terms[0].applyFilter": "true",
- "src_terms[0].match": "all_words",
- "src_terms[0].fields.siteContent": "true",
- "src_terms[0].fields.tags": "true",
- "src_terms[0].fields.files": "true",
- "src_sites.values": "ACMESPACE",
- "src_sitesForReferences.values": "systemsite",
- "src_languages.values": "en"
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/fda0d9ce/persistence-elasticsearch/core/src/test/resources/profile.json
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/test/resources/profile.json b/persistence-elasticsearch/core/src/test/resources/profile.json
deleted file mode 100644
index 8fc9d2c..0000000
--- a/persistence-elasticsearch/core/src/test/resources/profile.json
+++ /dev/null
@@ -1 +0,0 @@
-{"itemId":"6bb362d8-d094-44b4-ba2e-012163286825","itemType":"personaSession","scope":null,"profileId":"googleBot","profile":{"itemId":"googleBot","itemType":"persona","properties":{"description":"Represents a Google bot","firstName":"Google","lastName":"Bot"},"systemProperties":{},"segments":[],"scores":null,"mergedWith":null},"properties":{"operatingSystemName":"unknown","sessionCountryName":"United States","location":{"lon":-122.084058,"lat":37.422},"userAgentVersion":"2.1","sessionCountryCode":"US","deviceCategory":"Other","operatingSystemFamily":"unknown","userAgentName":"Googlebot/2.1","sessionCity":"Mountain View","remoteHost":"www.google.com","remoteAddr":"66.249.66.1"},"systemProperties":{},"timeStamp":"2014-09-18T09:18:52Z","lastEventDate":"2014-09-18T11:23:59Z","size":0,"duration":7507068}
\ No newline at end of file