You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2022/06/07 15:11:52 UTC
[unomi] 01/01: UNOMI-584: introduce new flattenedProperties for events, to allow mapping free event data to be stored in ElasticSearch
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch eventFlattenedProperties
in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 4ba96755ac120a7fc2f9efe735bc93cdeb9c9070
Author: Kevan <ke...@jahia.com>
AuthorDate: Tue Jun 7 17:11:32 2022 +0200
UNOMI-584: introduce new flattenedProperties for events, to allow mapping free event data to be stored in ElasticSearch
---
api/src/main/java/org/apache/unomi/api/Event.java | 20 ++++-
itests/pom.xml | 9 +-
.../java/org/apache/unomi/itests/JSONSchemaIT.java | 96 ++++++++++++++++++++--
.../schemas/event-flattened-invalid-1.json | 14 ++++
.../schemas/event-flattened-invalid-2.json | 16 ++++
.../schemas/event-flattened-invalid-3.json | 13 +++
.../resources/schemas/event-flattened-valid.json | 13 +++
.../src/test/resources/schemas/event-request.json | 6 ++
...ma-flattened-flattenedProperties-interests.json | 19 +++++
.../schema-flattened-flattenedProperties.json | 18 ++++
.../schemas/schema-flattened-properties.json | 18 ++++
.../test/resources/schemas/schema-flattened.json | 25 ++++++
persistence-elasticsearch/core/pom.xml | 7 ++
.../resources/META-INF/cxs/mappings/event.json | 3 +
pom.xml | 1 +
.../rest/service/impl/RestServiceUtilsImpl.java | 2 +
16 files changed, 269 insertions(+), 11 deletions(-)
diff --git a/api/src/main/java/org/apache/unomi/api/Event.java b/api/src/main/java/org/apache/unomi/api/Event.java
index 45a129609..3d418257c 100644
--- a/api/src/main/java/org/apache/unomi/api/Event.java
+++ b/api/src/main/java/org/apache/unomi/api/Event.java
@@ -58,6 +58,7 @@ public class Event extends Item implements TimestampedItem {
private String profileId = null;
private Date timeStamp;
private Map<String, Object> properties;
+ private Map<String, Object> flattenedProperties;
private transient Profile profile;
private transient Session session;
@@ -166,7 +167,8 @@ public class Event extends Item implements TimestampedItem {
}
this.timeStamp = timestamp;
- this.properties = new HashMap<String, Object>();
+ this.properties = new HashMap<>();
+ this.flattenedProperties = new HashMap<>();
actionPostExecutors = new ArrayList<>();
}
@@ -376,6 +378,22 @@ public class Event extends Item implements TimestampedItem {
this.properties = properties;
}
+ /**
+ * Retrieves the flattened properties
+ * @return the flattened properties.
+ */
+ public Map<String, Object> getFlattenedProperties() {
+ return flattenedProperties;
+ }
+
+ /**
+ * Set the flattened properties for current event
+ * @param flattenedProperties the properties
+ */
+ public void setFlattenedProperties(Map<String, Object> flattenedProperties) {
+ this.flattenedProperties = flattenedProperties;
+ }
+
/**
* @return the scope
*/
diff --git a/itests/pom.xml b/itests/pom.xml
index 1de0f6384..b65bce832 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -200,12 +200,12 @@
<groupId>com.github.alexcojocaru</groupId>
<artifactId>elasticsearch-maven-plugin</artifactId>
<!-- REPLACE THE FOLLOWING WITH THE PLUGIN VERSION YOU NEED -->
- <version>6.19</version>
+ <version>6.23</version>
<configuration>
<clusterName>contextElasticSearchITests</clusterName>
<transportPort>9500</transportPort>
<httpPort>9400</httpPort>
- <version>${elasticsearch.version}</version>
+ <version>${elasticsearch.test.version}</version>
<autoCreateIndex>true</autoCreateIndex>
<timeout>120</timeout>
<environmentVariables>
@@ -213,9 +213,10 @@
</environmentVariables>
<instanceSettings>
<properties>
- <cluster.routing.allocation.disk.threshold_enabled>false
- </cluster.routing.allocation.disk.threshold_enabled>
+ <cluster.routing.allocation.disk.threshold_enabled>false</cluster.routing.allocation.disk.threshold_enabled>
<http.cors.allow-origin>*</http.cors.allow-origin>
+ <http.cors.allow-methods>OPTIONS,HEAD,GET,POST,PUT,DELETE</http.cors.allow-methods>
+ <http.cors.allow-headers>Authorization,X-Requested-With,X-Auth-Token,Content-Type,Content-Length</http.cors.allow-headers>
</properties>
</instanceSettings>
</configuration>
diff --git a/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java b/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java
index 2600a8912..5640f211e 100644
--- a/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java
@@ -23,6 +23,8 @@ import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.util.EntityUtils;
import org.apache.unomi.api.Event;
+import org.apache.unomi.api.conditions.Condition;
+import org.apache.unomi.itests.tools.httpclient.HttpClientThatWaitsForUnomi;
import org.apache.unomi.schema.api.JsonSchemaWrapper;
import org.apache.unomi.schema.api.SchemaService;
import org.junit.After;
@@ -38,13 +40,9 @@ import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import java.io.IOException;
-import java.util.List;
-import java.util.Objects;
+import java.util.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
/**
* Class to tests the JSON schema features
@@ -54,6 +52,7 @@ import static org.junit.Assert.assertTrue;
@ExamReactorStrategy(PerSuite.class)
public class JSONSchemaIT extends BaseIT {
private final static Logger LOGGER = LoggerFactory.getLogger(JSONSchemaIT.class);
+ private final static String EVENT_COLLECTOR_URL = "/cxs/eventcollector";
private final static String JSONSCHEMA_URL = "/cxs/jsonSchema";
private static final int DEFAULT_TRYING_TIMEOUT = 2000;
private static final int DEFAULT_TRYING_TRIES = 30;
@@ -246,6 +245,53 @@ public class JSONSchemaIT extends BaseIT {
DEFAULT_TRYING_TRIES);
}
+ @Test
+ public void testFlattenedProperties() throws Exception {
+ assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/1-0-0"));
+ assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/properties/1-0-0"));
+ assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/properties/interests/1-0-0"));
+
+ // Test that at first the flattened event is not valid.
+ assertFalse(schemaService.isEventValid(resourceAsString("schemas/event-flattened-valid.json"), "flattened"));
+
+ // save schemas and check the event pass the validation
+ schemaService.saveSchema(resourceAsString("schemas/schema-flattened.json"));
+ schemaService.saveSchema(resourceAsString("schemas/schema-flattened-flattenedProperties.json"));
+ schemaService.saveSchema(resourceAsString("schemas/schema-flattened-flattenedProperties-interests.json"));
+ schemaService.saveSchema(resourceAsString("schemas/schema-flattened-properties.json"));
+ keepTrying("Event should be valid now",
+ () -> schemaService.isEventValid(resourceAsString("schemas/event-flattened-valid.json"), "flattened"),
+ isValid -> isValid, DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES);
+
+ // insure event is correctly indexed when send to /cxs/eventCollector
+ Event event = sendEventAndWaitItsIndexed("schemas/event-flattened-valid.json");
+ Map<String, Integer> interests = (Map<String, Integer>) event.getFlattenedProperties().get("interests");
+ assertEquals(15, interests.get("cars").intValue());
+ assertEquals(59, interests.get("football").intValue());
+ assertEquals(2, interests.size());
+
+ // check that range query is not working on flattened props:
+ Condition condition = new Condition(definitionsService.getConditionType("eventPropertyCondition"));
+ condition.setParameter("propertyName","flattenedProperties.interests.cars");
+ condition.setParameter("comparisonOperator","greaterThan");
+ condition.setParameter("propertyValueInteger", 2);
+ assertNull(persistenceService.query(condition, null, Event.class, 0, -1));
+
+ // check that term query is working on flattened props:
+ condition = new Condition(definitionsService.getConditionType("eventPropertyCondition"));
+ condition.setParameter("propertyName","flattenedProperties.interests.cars");
+ condition.setParameter("comparisonOperator","equals");
+ condition.setParameter("propertyValueInteger", 15);
+ List<Event> events = persistenceService.query(condition, null, Event.class, 0, -1).getList();
+ assertEquals(1, events.size());
+ assertEquals(events.get(0).getItemId(), event.getItemId());
+
+ // Bonus: Check that other invalid flattened events are correctly rejected by schema service:
+ assertFalse(schemaService.isEventValid(resourceAsString("schemas/event-flattened-invalid-1.json"), "flattened"));
+ assertFalse(schemaService.isEventValid(resourceAsString("schemas/event-flattened-invalid-2.json"), "flattened"));
+ assertFalse(schemaService.isEventValid(resourceAsString("schemas/event-flattened-invalid-3.json"), "flattened"));
+ }
+
@Test
public void testSaveFail_PredefinedJSONSchema() throws IOException {
try (CloseableHttpResponse response = post(JSONSCHEMA_URL, "schemas/schema-predefined.json", ContentType.TEXT_PLAIN)) {
@@ -266,4 +312,42 @@ public class JSONSchemaIT extends BaseIT {
assertEquals("Unable to save schema", 400, response.getStatusLine().getStatusCode());
}
}
+
+ private Event sendEventAndWaitItsIndexed(String eventResourcePath) throws IOException, InterruptedException {
+ // build event collector request
+ String eventMarker = UUID.randomUUID().toString();
+ HashMap<String, String> eventReplacements = new HashMap<>();
+ eventReplacements.put("EVENT_MARKER", eventMarker);
+ String event = getValidatedBundleJSON(eventResourcePath, eventReplacements);
+ HashMap<String, String> eventRequestReplacements = new HashMap<>();
+ eventRequestReplacements.put("EVENTS", event);
+ String eventRequest = getValidatedBundleJSON("schemas/event-request.json", eventRequestReplacements);
+
+ // send event
+ eventCollectorPost(eventRequest);
+
+ // wait for the event to be indexed
+ Condition condition = new Condition(definitionsService.getConditionType("eventPropertyCondition"));
+ condition.setParameter("propertyName","properties.marker.keyword");
+ condition.setParameter("comparisonOperator","equals");
+ condition.setParameter("propertyValue", eventMarker);
+ List<Event> events = keepTrying("The event should have been persisted",
+ () -> persistenceService.query(condition, null, Event.class), results -> results.size() == 1,
+ DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES);
+ return events.get(0);
+ }
+
+ private void eventCollectorPost(String eventCollectorRequest) {
+ HttpPost request = new HttpPost(URL + EVENT_COLLECTOR_URL);
+ request.addHeader("Content-Type", "application/json");
+ request.setEntity(new StringEntity(eventCollectorRequest, ContentType.create("application/json")));
+ CloseableHttpResponse response;
+ try {
+ response = HttpClientThatWaitsForUnomi.doRequest(request, 200);
+ } catch (Exception e) {
+ fail("Something went wrong with the request to Unomi that is unexpected: " + e.getMessage());
+ return;
+ }
+ assertEquals("Invalid response code", 200, response.getStatusLine().getStatusCode());
+ }
}
diff --git a/itests/src/test/resources/schemas/event-flattened-invalid-1.json b/itests/src/test/resources/schemas/event-flattened-invalid-1.json
new file mode 100644
index 000000000..ffc6eaa8c
--- /dev/null
+++ b/itests/src/test/resources/schemas/event-flattened-invalid-1.json
@@ -0,0 +1,14 @@
+{
+ "eventType":"flattened",
+ "scope":"scope",
+ "flattenedProperties": {
+ "interests": {
+ "cars": 15,
+ "football": 59
+ },
+ "notAllowedProp": "notAllowedProp"
+ },
+ "properties": {
+ "marker": "###EVENT_MARKER###"
+ }
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/schemas/event-flattened-invalid-2.json b/itests/src/test/resources/schemas/event-flattened-invalid-2.json
new file mode 100644
index 000000000..e062a1de6
--- /dev/null
+++ b/itests/src/test/resources/schemas/event-flattened-invalid-2.json
@@ -0,0 +1,16 @@
+{
+ "eventType":"flattened",
+ "scope":"scope",
+ "flattenedProperties": {
+ "interests": {
+ "cars": 15,
+ "football": 59,
+ "tennis": 42,
+ "fishing": 8,
+ "too_much_interests": 9
+ }
+ },
+ "properties": {
+ "marker": "###EVENT_MARKER###"
+ }
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/schemas/event-flattened-invalid-3.json b/itests/src/test/resources/schemas/event-flattened-invalid-3.json
new file mode 100644
index 000000000..a49597982
--- /dev/null
+++ b/itests/src/test/resources/schemas/event-flattened-invalid-3.json
@@ -0,0 +1,13 @@
+{
+ "eventType":"flattened",
+ "scope":"scope",
+ "flattenedProperties": {
+ "interests": {
+ "cars": 15,
+ "football": "not authorized value"
+ }
+ },
+ "properties": {
+ "marker": "###EVENT_MARKER###"
+ }
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/schemas/event-flattened-valid.json b/itests/src/test/resources/schemas/event-flattened-valid.json
new file mode 100644
index 000000000..0a9a80276
--- /dev/null
+++ b/itests/src/test/resources/schemas/event-flattened-valid.json
@@ -0,0 +1,13 @@
+{
+ "eventType":"flattened",
+ "scope":"scope",
+ "flattenedProperties": {
+ "interests": {
+ "cars": 15,
+ "football": 59
+ }
+ },
+ "properties": {
+ "marker": "###EVENT_MARKER###"
+ }
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/schemas/event-request.json b/itests/src/test/resources/schemas/event-request.json
new file mode 100644
index 000000000..d6a8c6f42
--- /dev/null
+++ b/itests/src/test/resources/schemas/event-request.json
@@ -0,0 +1,6 @@
+{
+ "sessionId": "dummy-session-id",
+ "events":[
+ ###EVENTS###
+ ]
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/schemas/schema-flattened-flattenedProperties-interests.json b/itests/src/test/resources/schemas/schema-flattened-flattenedProperties-interests.json
new file mode 100644
index 000000000..a5fde38fc
--- /dev/null
+++ b/itests/src/test/resources/schemas/schema-flattened-flattenedProperties-interests.json
@@ -0,0 +1,19 @@
+{
+ "$id": "https://vendor.test.com/schemas/json/events/flattened/flattenedProperties/interests/1-0-0",
+ "$schema": "https://json-schema.org/draft/2019-09/schema",
+ "self": {
+ "vendor": "com.vendor.test",
+ "name": "flattenedPropertiesInterests",
+ "format": "jsonschema",
+ "version": "1-0-0"
+ },
+ "title": "Flattened Properties interests",
+ "type": "object",
+ "patternProperties": {
+ ".": {
+ "type": "number"
+ }
+ },
+ "maxProperties": 4,
+ "unevaluatedProperties": false
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/schemas/schema-flattened-flattenedProperties.json b/itests/src/test/resources/schemas/schema-flattened-flattenedProperties.json
new file mode 100644
index 000000000..e50e883f2
--- /dev/null
+++ b/itests/src/test/resources/schemas/schema-flattened-flattenedProperties.json
@@ -0,0 +1,18 @@
+{
+ "$id": "https://vendor.test.com/schemas/json/events/flattened/flattenedProperties/1-0-0",
+ "$schema": "https://json-schema.org/draft/2019-09/schema",
+ "self": {
+ "vendor": "com.vendor.test",
+ "name": "flattenedProperties",
+ "format": "jsonschema",
+ "version": "1-0-0"
+ },
+ "title": "Flattened Properties",
+ "type": "object",
+ "properties": {
+ "interests": {
+ "$ref": "https://vendor.test.com/schemas/json/events/flattened/flattenedProperties/interests/1-0-0"
+ }
+ },
+ "unevaluatedProperties": false
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/schemas/schema-flattened-properties.json b/itests/src/test/resources/schemas/schema-flattened-properties.json
new file mode 100644
index 000000000..7d7fd7c0d
--- /dev/null
+++ b/itests/src/test/resources/schemas/schema-flattened-properties.json
@@ -0,0 +1,18 @@
+{
+ "$id": "https://vendor.test.com/schemas/json/events/flattened/properties/1-0-0",
+ "$schema": "https://json-schema.org/draft/2019-09/schema",
+ "self": {
+ "vendor": "com.vendor.test",
+ "name": "flattenedProperties",
+ "format": "jsonschema",
+ "version": "1-0-0"
+ },
+ "title": "flattenedProperties",
+ "type": "object",
+ "properties": {
+ "marker": {
+ "type": "string"
+ }
+ },
+ "unevaluatedProperties": false
+}
diff --git a/itests/src/test/resources/schemas/schema-flattened.json b/itests/src/test/resources/schemas/schema-flattened.json
new file mode 100644
index 000000000..0102486d6
--- /dev/null
+++ b/itests/src/test/resources/schemas/schema-flattened.json
@@ -0,0 +1,25 @@
+{
+ "$id": "https://vendor.test.com/schemas/json/events/flattened/1-0-0",
+ "$schema": "https://json-schema.org/draft/2019-09/schema",
+ "self":{
+ "vendor":"com.vendor.test",
+ "name":"flattened",
+ "format":"jsonschema",
+ "target":"events",
+ "version":"1-0-0"
+ },
+ "title": "Flattened Event test",
+ "type": "object",
+ "allOf": [
+ { "$ref": "https://unomi.apache.org/schemas/json/event/1-0-0" }
+ ],
+ "properties": {
+ "flattenedProperties": {
+ "$ref": "https://vendor.test.com/schemas/json/events/flattened/flattenedProperties/1-0-0"
+ },
+ "properties": {
+ "$ref": "https://vendor.test.com/schemas/json/events/flattened/properties/1-0-0"
+ }
+ },
+ "unevaluatedProperties": false
+}
diff --git a/persistence-elasticsearch/core/pom.xml b/persistence-elasticsearch/core/pom.xml
index beaf2de14..4b7791ad5 100644
--- a/persistence-elasticsearch/core/pom.xml
+++ b/persistence-elasticsearch/core/pom.xml
@@ -114,6 +114,13 @@
<artifactId>lucene-test-framework</artifactId>
<version>8.2.0</version>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <!-- internal dep that is scoped "compile", ending up to be embedded and conflicting with ES Rest client internal deps -->
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
index f3958e6ea..e7a8231b8 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
@@ -18,6 +18,9 @@
}
],
"properties": {
+ "flattenedProperties": {
+ "type": "flattened"
+ },
"timeStamp": {
"type": "date"
},
diff --git a/pom.xml b/pom.xml
index 89d6d038b..ee5abe5f9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -72,6 +72,7 @@
<version.karaf.cellar>4.2.1</version.karaf.cellar>
<version.pax.exam>4.13.1</version.pax.exam>
<elasticsearch.version>7.4.2</elasticsearch.version>
+ <elasticsearch.test.version>7.11.0</elasticsearch.test.version>
<groovy.version>3.0.3</groovy.version>
<networknt.version>1.0.49</networknt.version>
<bean.validation.version>1.1.0.Final</bean.validation.version>
diff --git a/rest/src/main/java/org/apache/unomi/rest/service/impl/RestServiceUtilsImpl.java b/rest/src/main/java/org/apache/unomi/rest/service/impl/RestServiceUtilsImpl.java
index ac617d3d7..44612f158 100644
--- a/rest/src/main/java/org/apache/unomi/rest/service/impl/RestServiceUtilsImpl.java
+++ b/rest/src/main/java/org/apache/unomi/rest/service/impl/RestServiceUtilsImpl.java
@@ -93,6 +93,7 @@ public class RestServiceUtilsImpl implements RestServiceUtils {
if (event.getEventType() != null) {
Event eventToSend = new Event(event.getEventType(), session, profile, event.getSourceId(), event.getSource(),
event.getTarget(), event.getProperties(), timestamp, event.isPersistent());
+ eventToSend.setFlattenedProperties(event.getFlattenedProperties());
if (!eventService.isEventAllowed(event, thirdPartyId)) {
logger.warn("Event is not allowed : {}", event.getEventType());
continue;
@@ -100,6 +101,7 @@ public class RestServiceUtilsImpl implements RestServiceUtils {
if (thirdPartyId != null && event.getItemId() != null) {
eventToSend = new Event(event.getItemId(), event.getEventType(), session, profile, event.getSourceId(),
event.getSource(), event.getTarget(), event.getProperties(), timestamp, event.isPersistent());
+ eventToSend.setFlattenedProperties(event.getFlattenedProperties());
}
if (filteredEventTypes != null && filteredEventTypes.contains(event.getEventType())) {
logger.debug("Profile is filtering event type {}", event.getEventType());