You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2022/06/07 15:11:51 UTC
[unomi] branch eventFlattenedProperties created (now 4ba96755a)
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a change to branch eventFlattenedProperties
in repository https://gitbox.apache.org/repos/asf/unomi.git
at 4ba96755a UNOMI-584: introduce new flattenedProperties for events, to allow mapping free event data to be stored in ElasticSearch
This branch includes the following new commits:
new 4ba96755a UNOMI-584: introduce new flattenedProperties for events, to allow mapping free event data to be stored in ElasticSearch
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[unomi] 01/01: UNOMI-584: introduce new flattenedProperties for events, to allow mapping free event data to be stored in ElasticSearch
Posted by jk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch eventFlattenedProperties
in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 4ba96755ac120a7fc2f9efe735bc93cdeb9c9070
Author: Kevan <ke...@jahia.com>
AuthorDate: Tue Jun 7 17:11:32 2022 +0200
UNOMI-584: introduce new flattenedProperties for events, to allow mapping free event data to be stored in ElasticSearch
---
api/src/main/java/org/apache/unomi/api/Event.java | 20 ++++-
itests/pom.xml | 9 +-
.../java/org/apache/unomi/itests/JSONSchemaIT.java | 96 ++++++++++++++++++++--
.../schemas/event-flattened-invalid-1.json | 14 ++++
.../schemas/event-flattened-invalid-2.json | 16 ++++
.../schemas/event-flattened-invalid-3.json | 13 +++
.../resources/schemas/event-flattened-valid.json | 13 +++
.../src/test/resources/schemas/event-request.json | 6 ++
...ma-flattened-flattenedProperties-interests.json | 19 +++++
.../schema-flattened-flattenedProperties.json | 18 ++++
.../schemas/schema-flattened-properties.json | 18 ++++
.../test/resources/schemas/schema-flattened.json | 25 ++++++
persistence-elasticsearch/core/pom.xml | 7 ++
.../resources/META-INF/cxs/mappings/event.json | 3 +
pom.xml | 1 +
.../rest/service/impl/RestServiceUtilsImpl.java | 2 +
16 files changed, 269 insertions(+), 11 deletions(-)
diff --git a/api/src/main/java/org/apache/unomi/api/Event.java b/api/src/main/java/org/apache/unomi/api/Event.java
index 45a129609..3d418257c 100644
--- a/api/src/main/java/org/apache/unomi/api/Event.java
+++ b/api/src/main/java/org/apache/unomi/api/Event.java
@@ -58,6 +58,7 @@ public class Event extends Item implements TimestampedItem {
private String profileId = null;
private Date timeStamp;
private Map<String, Object> properties;
+ private Map<String, Object> flattenedProperties;
private transient Profile profile;
private transient Session session;
@@ -166,7 +167,8 @@ public class Event extends Item implements TimestampedItem {
}
this.timeStamp = timestamp;
- this.properties = new HashMap<String, Object>();
+ this.properties = new HashMap<>();
+ this.flattenedProperties = new HashMap<>();
actionPostExecutors = new ArrayList<>();
}
@@ -376,6 +378,22 @@ public class Event extends Item implements TimestampedItem {
this.properties = properties;
}
+ /**
+ * Retrieves the flattened properties
+ * @return the flattened properties.
+ */
+ public Map<String, Object> getFlattenedProperties() {
+ return flattenedProperties;
+ }
+
+ /**
+ * Set the flattened properties for current event
+ * @param flattenedProperties the properties
+ */
+ public void setFlattenedProperties(Map<String, Object> flattenedProperties) {
+ this.flattenedProperties = flattenedProperties;
+ }
+
/**
* @return the scope
*/
diff --git a/itests/pom.xml b/itests/pom.xml
index 1de0f6384..b65bce832 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -200,12 +200,12 @@
<groupId>com.github.alexcojocaru</groupId>
<artifactId>elasticsearch-maven-plugin</artifactId>
<!-- REPLACE THE FOLLOWING WITH THE PLUGIN VERSION YOU NEED -->
- <version>6.19</version>
+ <version>6.23</version>
<configuration>
<clusterName>contextElasticSearchITests</clusterName>
<transportPort>9500</transportPort>
<httpPort>9400</httpPort>
- <version>${elasticsearch.version}</version>
+ <version>${elasticsearch.test.version}</version>
<autoCreateIndex>true</autoCreateIndex>
<timeout>120</timeout>
<environmentVariables>
@@ -213,9 +213,10 @@
</environmentVariables>
<instanceSettings>
<properties>
- <cluster.routing.allocation.disk.threshold_enabled>false
- </cluster.routing.allocation.disk.threshold_enabled>
+ <cluster.routing.allocation.disk.threshold_enabled>false</cluster.routing.allocation.disk.threshold_enabled>
<http.cors.allow-origin>*</http.cors.allow-origin>
+ <http.cors.allow-methods>OPTIONS,HEAD,GET,POST,PUT,DELETE</http.cors.allow-methods>
+ <http.cors.allow-headers>Authorization,X-Requested-With,X-Auth-Token,Content-Type,Content-Length</http.cors.allow-headers>
</properties>
</instanceSettings>
</configuration>
diff --git a/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java b/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java
index 2600a8912..5640f211e 100644
--- a/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java
@@ -23,6 +23,8 @@ import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.util.EntityUtils;
import org.apache.unomi.api.Event;
+import org.apache.unomi.api.conditions.Condition;
+import org.apache.unomi.itests.tools.httpclient.HttpClientThatWaitsForUnomi;
import org.apache.unomi.schema.api.JsonSchemaWrapper;
import org.apache.unomi.schema.api.SchemaService;
import org.junit.After;
@@ -38,13 +40,9 @@ import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import java.io.IOException;
-import java.util.List;
-import java.util.Objects;
+import java.util.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
/**
* Class to tests the JSON schema features
@@ -54,6 +52,7 @@ import static org.junit.Assert.assertTrue;
@ExamReactorStrategy(PerSuite.class)
public class JSONSchemaIT extends BaseIT {
private final static Logger LOGGER = LoggerFactory.getLogger(JSONSchemaIT.class);
+ private final static String EVENT_COLLECTOR_URL = "/cxs/eventcollector";
private final static String JSONSCHEMA_URL = "/cxs/jsonSchema";
private static final int DEFAULT_TRYING_TIMEOUT = 2000;
private static final int DEFAULT_TRYING_TRIES = 30;
@@ -246,6 +245,53 @@ public class JSONSchemaIT extends BaseIT {
DEFAULT_TRYING_TRIES);
}
+ @Test
+ public void testFlattenedProperties() throws Exception {
+ assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/1-0-0"));
+ assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/properties/1-0-0"));
+ assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/properties/interests/1-0-0"));
+
+ // Test that at first the flattened event is not valid.
+ assertFalse(schemaService.isEventValid(resourceAsString("schemas/event-flattened-valid.json"), "flattened"));
+
+ // save schemas and check the event pass the validation
+ schemaService.saveSchema(resourceAsString("schemas/schema-flattened.json"));
+ schemaService.saveSchema(resourceAsString("schemas/schema-flattened-flattenedProperties.json"));
+ schemaService.saveSchema(resourceAsString("schemas/schema-flattened-flattenedProperties-interests.json"));
+ schemaService.saveSchema(resourceAsString("schemas/schema-flattened-properties.json"));
+ keepTrying("Event should be valid now",
+ () -> schemaService.isEventValid(resourceAsString("schemas/event-flattened-valid.json"), "flattened"),
+ isValid -> isValid, DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES);
+
+ // insure event is correctly indexed when send to /cxs/eventCollector
+ Event event = sendEventAndWaitItsIndexed("schemas/event-flattened-valid.json");
+ Map<String, Integer> interests = (Map<String, Integer>) event.getFlattenedProperties().get("interests");
+ assertEquals(15, interests.get("cars").intValue());
+ assertEquals(59, interests.get("football").intValue());
+ assertEquals(2, interests.size());
+
+ // check that range query is not working on flattened props:
+ Condition condition = new Condition(definitionsService.getConditionType("eventPropertyCondition"));
+ condition.setParameter("propertyName","flattenedProperties.interests.cars");
+ condition.setParameter("comparisonOperator","greaterThan");
+ condition.setParameter("propertyValueInteger", 2);
+ assertNull(persistenceService.query(condition, null, Event.class, 0, -1));
+
+ // check that term query is working on flattened props:
+ condition = new Condition(definitionsService.getConditionType("eventPropertyCondition"));
+ condition.setParameter("propertyName","flattenedProperties.interests.cars");
+ condition.setParameter("comparisonOperator","equals");
+ condition.setParameter("propertyValueInteger", 15);
+ List<Event> events = persistenceService.query(condition, null, Event.class, 0, -1).getList();
+ assertEquals(1, events.size());
+ assertEquals(events.get(0).getItemId(), event.getItemId());
+
+ // Bonus: Check that other invalid flattened events are correctly rejected by schema service:
+ assertFalse(schemaService.isEventValid(resourceAsString("schemas/event-flattened-invalid-1.json"), "flattened"));
+ assertFalse(schemaService.isEventValid(resourceAsString("schemas/event-flattened-invalid-2.json"), "flattened"));
+ assertFalse(schemaService.isEventValid(resourceAsString("schemas/event-flattened-invalid-3.json"), "flattened"));
+ }
+
@Test
public void testSaveFail_PredefinedJSONSchema() throws IOException {
try (CloseableHttpResponse response = post(JSONSCHEMA_URL, "schemas/schema-predefined.json", ContentType.TEXT_PLAIN)) {
@@ -266,4 +312,42 @@ public class JSONSchemaIT extends BaseIT {
assertEquals("Unable to save schema", 400, response.getStatusLine().getStatusCode());
}
}
+
+ private Event sendEventAndWaitItsIndexed(String eventResourcePath) throws IOException, InterruptedException {
+ // build event collector request
+ String eventMarker = UUID.randomUUID().toString();
+ HashMap<String, String> eventReplacements = new HashMap<>();
+ eventReplacements.put("EVENT_MARKER", eventMarker);
+ String event = getValidatedBundleJSON(eventResourcePath, eventReplacements);
+ HashMap<String, String> eventRequestReplacements = new HashMap<>();
+ eventRequestReplacements.put("EVENTS", event);
+ String eventRequest = getValidatedBundleJSON("schemas/event-request.json", eventRequestReplacements);
+
+ // send event
+ eventCollectorPost(eventRequest);
+
+ // wait for the event to be indexed
+ Condition condition = new Condition(definitionsService.getConditionType("eventPropertyCondition"));
+ condition.setParameter("propertyName","properties.marker.keyword");
+ condition.setParameter("comparisonOperator","equals");
+ condition.setParameter("propertyValue", eventMarker);
+ List<Event> events = keepTrying("The event should have been persisted",
+ () -> persistenceService.query(condition, null, Event.class), results -> results.size() == 1,
+ DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES);
+ return events.get(0);
+ }
+
+ private void eventCollectorPost(String eventCollectorRequest) {
+ HttpPost request = new HttpPost(URL + EVENT_COLLECTOR_URL);
+ request.addHeader("Content-Type", "application/json");
+ request.setEntity(new StringEntity(eventCollectorRequest, ContentType.create("application/json")));
+ CloseableHttpResponse response;
+ try {
+ response = HttpClientThatWaitsForUnomi.doRequest(request, 200);
+ } catch (Exception e) {
+ fail("Something went wrong with the request to Unomi that is unexpected: " + e.getMessage());
+ return;
+ }
+ assertEquals("Invalid response code", 200, response.getStatusLine().getStatusCode());
+ }
}
diff --git a/itests/src/test/resources/schemas/event-flattened-invalid-1.json b/itests/src/test/resources/schemas/event-flattened-invalid-1.json
new file mode 100644
index 000000000..ffc6eaa8c
--- /dev/null
+++ b/itests/src/test/resources/schemas/event-flattened-invalid-1.json
@@ -0,0 +1,14 @@
+{
+ "eventType":"flattened",
+ "scope":"scope",
+ "flattenedProperties": {
+ "interests": {
+ "cars": 15,
+ "football": 59
+ },
+ "notAllowedProp": "notAllowedProp"
+ },
+ "properties": {
+ "marker": "###EVENT_MARKER###"
+ }
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/schemas/event-flattened-invalid-2.json b/itests/src/test/resources/schemas/event-flattened-invalid-2.json
new file mode 100644
index 000000000..e062a1de6
--- /dev/null
+++ b/itests/src/test/resources/schemas/event-flattened-invalid-2.json
@@ -0,0 +1,16 @@
+{
+ "eventType":"flattened",
+ "scope":"scope",
+ "flattenedProperties": {
+ "interests": {
+ "cars": 15,
+ "football": 59,
+ "tennis": 42,
+ "fishing": 8,
+ "too_much_interests": 9
+ }
+ },
+ "properties": {
+ "marker": "###EVENT_MARKER###"
+ }
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/schemas/event-flattened-invalid-3.json b/itests/src/test/resources/schemas/event-flattened-invalid-3.json
new file mode 100644
index 000000000..a49597982
--- /dev/null
+++ b/itests/src/test/resources/schemas/event-flattened-invalid-3.json
@@ -0,0 +1,13 @@
+{
+ "eventType":"flattened",
+ "scope":"scope",
+ "flattenedProperties": {
+ "interests": {
+ "cars": 15,
+ "football": "not authorized value"
+ }
+ },
+ "properties": {
+ "marker": "###EVENT_MARKER###"
+ }
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/schemas/event-flattened-valid.json b/itests/src/test/resources/schemas/event-flattened-valid.json
new file mode 100644
index 000000000..0a9a80276
--- /dev/null
+++ b/itests/src/test/resources/schemas/event-flattened-valid.json
@@ -0,0 +1,13 @@
+{
+ "eventType":"flattened",
+ "scope":"scope",
+ "flattenedProperties": {
+ "interests": {
+ "cars": 15,
+ "football": 59
+ }
+ },
+ "properties": {
+ "marker": "###EVENT_MARKER###"
+ }
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/schemas/event-request.json b/itests/src/test/resources/schemas/event-request.json
new file mode 100644
index 000000000..d6a8c6f42
--- /dev/null
+++ b/itests/src/test/resources/schemas/event-request.json
@@ -0,0 +1,6 @@
+{
+ "sessionId": "dummy-session-id",
+ "events":[
+ ###EVENTS###
+ ]
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/schemas/schema-flattened-flattenedProperties-interests.json b/itests/src/test/resources/schemas/schema-flattened-flattenedProperties-interests.json
new file mode 100644
index 000000000..a5fde38fc
--- /dev/null
+++ b/itests/src/test/resources/schemas/schema-flattened-flattenedProperties-interests.json
@@ -0,0 +1,19 @@
+{
+ "$id": "https://vendor.test.com/schemas/json/events/flattened/flattenedProperties/interests/1-0-0",
+ "$schema": "https://json-schema.org/draft/2019-09/schema",
+ "self": {
+ "vendor": "com.vendor.test",
+ "name": "flattenedPropertiesInterests",
+ "format": "jsonschema",
+ "version": "1-0-0"
+ },
+ "title": "Flattened Properties interests",
+ "type": "object",
+ "patternProperties": {
+ ".": {
+ "type": "number"
+ }
+ },
+ "maxProperties": 4,
+ "unevaluatedProperties": false
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/schemas/schema-flattened-flattenedProperties.json b/itests/src/test/resources/schemas/schema-flattened-flattenedProperties.json
new file mode 100644
index 000000000..e50e883f2
--- /dev/null
+++ b/itests/src/test/resources/schemas/schema-flattened-flattenedProperties.json
@@ -0,0 +1,18 @@
+{
+ "$id": "https://vendor.test.com/schemas/json/events/flattened/flattenedProperties/1-0-0",
+ "$schema": "https://json-schema.org/draft/2019-09/schema",
+ "self": {
+ "vendor": "com.vendor.test",
+ "name": "flattenedProperties",
+ "format": "jsonschema",
+ "version": "1-0-0"
+ },
+ "title": "Flattened Properties",
+ "type": "object",
+ "properties": {
+ "interests": {
+ "$ref": "https://vendor.test.com/schemas/json/events/flattened/flattenedProperties/interests/1-0-0"
+ }
+ },
+ "unevaluatedProperties": false
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/schemas/schema-flattened-properties.json b/itests/src/test/resources/schemas/schema-flattened-properties.json
new file mode 100644
index 000000000..7d7fd7c0d
--- /dev/null
+++ b/itests/src/test/resources/schemas/schema-flattened-properties.json
@@ -0,0 +1,18 @@
+{
+ "$id": "https://vendor.test.com/schemas/json/events/flattened/properties/1-0-0",
+ "$schema": "https://json-schema.org/draft/2019-09/schema",
+ "self": {
+ "vendor": "com.vendor.test",
+ "name": "flattenedProperties",
+ "format": "jsonschema",
+ "version": "1-0-0"
+ },
+ "title": "flattenedProperties",
+ "type": "object",
+ "properties": {
+ "marker": {
+ "type": "string"
+ }
+ },
+ "unevaluatedProperties": false
+}
diff --git a/itests/src/test/resources/schemas/schema-flattened.json b/itests/src/test/resources/schemas/schema-flattened.json
new file mode 100644
index 000000000..0102486d6
--- /dev/null
+++ b/itests/src/test/resources/schemas/schema-flattened.json
@@ -0,0 +1,25 @@
+{
+ "$id": "https://vendor.test.com/schemas/json/events/flattened/1-0-0",
+ "$schema": "https://json-schema.org/draft/2019-09/schema",
+ "self":{
+ "vendor":"com.vendor.test",
+ "name":"flattened",
+ "format":"jsonschema",
+ "target":"events",
+ "version":"1-0-0"
+ },
+ "title": "Flattened Event test",
+ "type": "object",
+ "allOf": [
+ { "$ref": "https://unomi.apache.org/schemas/json/event/1-0-0" }
+ ],
+ "properties": {
+ "flattenedProperties": {
+ "$ref": "https://vendor.test.com/schemas/json/events/flattened/flattenedProperties/1-0-0"
+ },
+ "properties": {
+ "$ref": "https://vendor.test.com/schemas/json/events/flattened/properties/1-0-0"
+ }
+ },
+ "unevaluatedProperties": false
+}
diff --git a/persistence-elasticsearch/core/pom.xml b/persistence-elasticsearch/core/pom.xml
index beaf2de14..4b7791ad5 100644
--- a/persistence-elasticsearch/core/pom.xml
+++ b/persistence-elasticsearch/core/pom.xml
@@ -114,6 +114,13 @@
<artifactId>lucene-test-framework</artifactId>
<version>8.2.0</version>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <!-- internal dep that is scoped "compile", ending up to be embedded and conflicting with ES Rest client internal deps -->
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
index f3958e6ea..e7a8231b8 100644
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
@@ -18,6 +18,9 @@
}
],
"properties": {
+ "flattenedProperties": {
+ "type": "flattened"
+ },
"timeStamp": {
"type": "date"
},
diff --git a/pom.xml b/pom.xml
index 89d6d038b..ee5abe5f9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -72,6 +72,7 @@
<version.karaf.cellar>4.2.1</version.karaf.cellar>
<version.pax.exam>4.13.1</version.pax.exam>
<elasticsearch.version>7.4.2</elasticsearch.version>
+ <elasticsearch.test.version>7.11.0</elasticsearch.test.version>
<groovy.version>3.0.3</groovy.version>
<networknt.version>1.0.49</networknt.version>
<bean.validation.version>1.1.0.Final</bean.validation.version>
diff --git a/rest/src/main/java/org/apache/unomi/rest/service/impl/RestServiceUtilsImpl.java b/rest/src/main/java/org/apache/unomi/rest/service/impl/RestServiceUtilsImpl.java
index ac617d3d7..44612f158 100644
--- a/rest/src/main/java/org/apache/unomi/rest/service/impl/RestServiceUtilsImpl.java
+++ b/rest/src/main/java/org/apache/unomi/rest/service/impl/RestServiceUtilsImpl.java
@@ -93,6 +93,7 @@ public class RestServiceUtilsImpl implements RestServiceUtils {
if (event.getEventType() != null) {
Event eventToSend = new Event(event.getEventType(), session, profile, event.getSourceId(), event.getSource(),
event.getTarget(), event.getProperties(), timestamp, event.isPersistent());
+ eventToSend.setFlattenedProperties(event.getFlattenedProperties());
if (!eventService.isEventAllowed(event, thirdPartyId)) {
logger.warn("Event is not allowed : {}", event.getEventType());
continue;
@@ -100,6 +101,7 @@ public class RestServiceUtilsImpl implements RestServiceUtils {
if (thirdPartyId != null && event.getItemId() != null) {
eventToSend = new Event(event.getItemId(), event.getEventType(), session, profile, event.getSourceId(),
event.getSource(), event.getTarget(), event.getProperties(), timestamp, event.isPersistent());
+ eventToSend.setFlattenedProperties(event.getFlattenedProperties());
}
if (filteredEventTypes != null && filteredEventTypes.contains(event.getEventType())) {
logger.debug("Profile is filtering event type {}", event.getEventType());