You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by js...@apache.org on 2023/05/03 08:41:02 UTC
[unomi] branch master updated: Unomi 775 add validation endpoint (#612)
This is an automated email from the ASF dual-hosted git repository.
jsinovassinnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new 10f90be86 Unomi 775 add validation endpoint (#612)
10f90be86 is described below
commit 10f90be860f1655c9216fef49a46cae047d63be5
Author: jsinovassin <58...@users.noreply.github.com>
AuthorDate: Wed May 3 09:40:57 2023 +0100
Unomi 775 add validation endpoint (#612)
* UNOMI-775 : fix validation message comparison
* UNOMI-775 : add json schema validation endpoint for event list
* UNOMI-775 : add documentation
---
.../unomi/schema/rest/JsonSchemaEndPoint.java | 27 +++++++--
.../org/apache/unomi/schema/api/SchemaService.java | 10 ++++
.../apache/unomi/schema/api/ValidationError.java | 13 ++---
.../unomi/schema/api/ValidationException.java | 1 +
.../unomi/schema/impl/SchemaServiceImpl.java | 62 +++++++++++++++++----
.../java/org/apache/unomi/itests/JSONSchemaIT.java | 64 +++++++++++++++++++---
.../asciidoc/jsonSchema/json-schema-develop.adoc | 60 ++++++++++++++++++++
7 files changed, 206 insertions(+), 31 deletions(-)
diff --git a/extensions/json-schema/rest/src/main/java/org/apache/unomi/schema/rest/JsonSchemaEndPoint.java b/extensions/json-schema/rest/src/main/java/org/apache/unomi/schema/rest/JsonSchemaEndPoint.java
index fdb919538..4064edf20 100644
--- a/extensions/json-schema/rest/src/main/java/org/apache/unomi/schema/rest/JsonSchemaEndPoint.java
+++ b/extensions/json-schema/rest/src/main/java/org/apache/unomi/schema/rest/JsonSchemaEndPoint.java
@@ -36,8 +36,7 @@ import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import java.util.Collection;
-import java.util.Set;
+import java.util.*;
@WebService
@Produces(MediaType.APPLICATION_JSON + ";charset=UTF-8")
@@ -95,7 +94,7 @@ public class JsonSchemaEndPoint {
*/
@POST
@Path("/")
- @Consumes({ MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON })
+ @Consumes({MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON})
@Produces(MediaType.APPLICATION_JSON)
public Response save(String jsonSchema) {
try {
@@ -119,12 +118,13 @@ public class JsonSchemaEndPoint {
/**
* Being able to validate a given event is useful when you want to develop custom events and associated schemas
+ *
* @param event the event to be validated
* @return Validation error messages if there is some
*/
@POST
@Produces(MediaType.APPLICATION_JSON + ";charset=UTF-8")
- @Consumes({ MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON })
+ @Consumes({MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON})
@Path("/validateEvent")
public Collection<ValidationError> validateEvent(String event) {
try {
@@ -134,4 +134,23 @@ public class JsonSchemaEndPoint {
throw new InvalidRequestException(errorMessage, errorMessage);
}
}
+
+ /**
+ * Being able to validate a given list of event is useful when you want to develop custom events and associated schemas
+ *
+ * @param events the events to be validated
+ * @return Validation error messages if there is some grouped per event type
+ */
+ @POST
+ @Produces(MediaType.APPLICATION_JSON + ";charset=UTF-8")
+ @Consumes({MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON})
+ @Path("/validateEvents")
+ public Map<String, Set<ValidationError>> validateEvents(String events) {
+ try {
+ return schemaService.validateEvents(events);
+ } catch (Exception e) {
+ String errorMessage = "Unable to validate events: " + e.getMessage();
+ throw new InvalidRequestException(errorMessage, errorMessage);
+ }
+ }
}
diff --git a/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/api/SchemaService.java b/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/api/SchemaService.java
index 2690f5ba8..162ecbe77 100644
--- a/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/api/SchemaService.java
+++ b/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/api/SchemaService.java
@@ -20,6 +20,7 @@ package org.apache.unomi.schema.api;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
+import java.util.Map;
import java.util.Set;
/**
@@ -63,6 +64,15 @@ public interface SchemaService {
*/
Set<ValidationError> validateEvent(String event) throws ValidationException;
+ /**
+ * perform a validation of a list of the given events
+ *
+ * @param events the events to validate
+ * @return The Map of validation errors group per event type in case there is some, empty map otherwise
+ * @throws ValidationException in case something goes wrong and validation could not be performed.
+ */
+ Map<String,Set<ValidationError>> validateEvents(String events) throws ValidationException;
+
/**
* Get the list of installed Json Schema Ids
*
diff --git a/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/api/ValidationError.java b/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/api/ValidationError.java
index 85aef8afb..7adfabef1 100644
--- a/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/api/ValidationError.java
+++ b/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/api/ValidationError.java
@@ -27,22 +27,19 @@ import java.io.Serializable;
*/
public class ValidationError implements Serializable {
- private transient final ValidationMessage validationMessage;
+ private transient final String validationMessage;
- public ValidationError(ValidationMessage validationMessage) {
+ public ValidationError(String validationMessage) {
this.validationMessage = validationMessage;
}
public String getError() {
- return validationMessage.getMessage();
- }
-
- public String toString() {
- return validationMessage.toString();
+ return validationMessage;
}
public boolean equals(Object o) {
- return validationMessage.equals(o);
+ ValidationError other = (ValidationError) o;
+ return validationMessage.equals(other.getError());
}
public int hashCode() {
diff --git a/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/api/ValidationException.java b/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/api/ValidationException.java
index d2a278907..58610b285 100644
--- a/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/api/ValidationException.java
+++ b/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/api/ValidationException.java
@@ -22,6 +22,7 @@ package org.apache.unomi.schema.api;
* Or when we can't perform the validation due to missing data or invalid required data
*/
public class ValidationException extends Exception {
+
public ValidationException(String message) {
super(message);
}
diff --git a/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/impl/SchemaServiceImpl.java b/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/impl/SchemaServiceImpl.java
index 2371738b6..ca54fd243 100644
--- a/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/impl/SchemaServiceImpl.java
+++ b/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/impl/SchemaServiceImpl.java
@@ -40,6 +40,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
+import java.text.MessageFormat;
import java.util.*;
import java.util.concurrent.*;
import java.util.stream.Collectors;
@@ -51,10 +52,12 @@ public class SchemaServiceImpl implements SchemaService {
private static final Logger logger = LoggerFactory.getLogger(SchemaServiceImpl.class.getName());
private static final String TARGET_EVENTS = "events";
+ private static final String GENERIC_ERROR_KEY = "error";
+
ObjectMapper objectMapper = new ObjectMapper();
/**
- * Schemas provided by Unomi runtime bundles in /META-INF/cxs/schemas/...
+ * Schemas provided by Unomi runtime bundles in /META-INF/cxs/schemas/...
*/
private final ConcurrentMap<String, JsonSchemaWrapper> predefinedUnomiJSONSchemaById = new ConcurrentHashMap<>();
/**
@@ -111,12 +114,48 @@ public class SchemaServiceImpl implements SchemaService {
@Override
public Set<ValidationError> validateEvent(String event) throws ValidationException {
- JsonNode jsonEvent = parseData(event);
- String eventType = extractEventType(jsonEvent);
- JsonSchemaWrapper eventSchema = getSchemaForEventType(eventType);
- JsonSchema jsonSchema = getJsonSchema(eventSchema.getItemId());
+ return validateEvents("[" + event + "]").values().stream()
+ .flatMap(Set::stream)
+ .collect(Collectors.toSet());
+ }
+
+ @Override
+ public Map<String, Set<ValidationError>> validateEvents(String events) throws ValidationException {
+ Map<String, Set<ValidationError>> errorsPerEventType = new HashMap<>();
+ JsonNode eventsNodes = parseData(events);
+ eventsNodes.forEach(event -> {
+ String eventType = null;
+ try {
+ eventType = extractEventType(event);
+ JsonSchemaWrapper eventSchema = getSchemaForEventType(eventType);
+ JsonSchema jsonSchema = getJsonSchema(eventSchema.getItemId());
+
+ Set<ValidationError> errors = validate(event, jsonSchema);
+ if (!errors.isEmpty()) {
+ if (errorsPerEventType.containsKey(eventType)) {
+ errorsPerEventType.get(eventType).addAll(errors);
+ } else {
+ errorsPerEventType.put(eventType, errors);
+ }
+ }
+ } catch (ValidationException e) {
+ Set<ValidationError> errors = buildCustomErrorMessage(e.getMessage());
+ String eventTypeOrErrorKey = eventType != null ? eventType : GENERIC_ERROR_KEY;
+ if (errorsPerEventType.containsKey(eventTypeOrErrorKey)) {
+ errorsPerEventType.get(eventTypeOrErrorKey).addAll(errors);
+ } else {
+ errorsPerEventType.put(eventTypeOrErrorKey, errors);
+ }
+ }
+ });
+ return errorsPerEventType;
+ }
- return validate(jsonEvent, jsonSchema);
+ private Set<ValidationError> buildCustomErrorMessage(String errorMessage) {
+ ValidationError error = new ValidationError(errorMessage);
+ Set<ValidationError> errors = new HashSet<>();
+ errors.add(error);
+ return errors;
}
@Override
@@ -145,9 +184,9 @@ public class SchemaServiceImpl implements SchemaService {
return schemasById.values().stream()
.filter(jsonSchemaWrapper ->
jsonSchemaWrapper.getTarget() != null &&
- jsonSchemaWrapper.getTarget().equals(TARGET_EVENTS) &&
- jsonSchemaWrapper.getName() != null &&
- jsonSchemaWrapper.getName().equals(eventType))
+ jsonSchemaWrapper.getTarget().equals(TARGET_EVENTS) &&
+ jsonSchemaWrapper.getName() != null &&
+ jsonSchemaWrapper.getName().equals(eventType))
.findFirst()
.orElseThrow(() -> new ValidationException("Schema not found for event type: " + eventType));
}
@@ -199,7 +238,7 @@ public class SchemaServiceImpl implements SchemaService {
return validationMessages != null ?
validationMessages.stream()
- .map(ValidationError::new)
+ .map(validationMessage -> new ValidationError(validationMessage.getMessage()))
.collect(Collectors.toSet()) :
Collections.emptySet();
} catch (Exception e) {
@@ -211,7 +250,6 @@ public class SchemaServiceImpl implements SchemaService {
if (StringUtils.isEmpty(data)) {
throw new ValidationException("Empty data, nothing to validate");
}
-
try {
return objectMapper.readTree(data);
} catch (Exception e) {
@@ -318,7 +356,7 @@ public class SchemaServiceImpl implements SchemaService {
ArrayNode allOf;
if (jsonSchema.at("/allOf") instanceof MissingNode) {
allOf = objectMapper.createArrayNode();
- } else if (jsonSchema.at("/allOf") instanceof ArrayNode){
+ } else if (jsonSchema.at("/allOf") instanceof ArrayNode) {
allOf = (ArrayNode) jsonSchema.at("/allOf");
} else {
logger.warn("Cannot extends schema allOf property, it should be an Array, please fix your schema definition for schema: {}", id);
diff --git a/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java b/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java
index 43cb5949d..366f3ece5 100644
--- a/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java
@@ -30,6 +30,7 @@ import org.apache.unomi.api.services.ScopeService;
import org.apache.unomi.itests.tools.httpclient.HttpClientThatWaitsForUnomi;
import org.apache.unomi.schema.api.JsonSchemaWrapper;
import org.apache.unomi.schema.api.SchemaService;
+import org.apache.unomi.schema.api.ValidationError;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -47,6 +48,8 @@ import java.util.*;
import static org.junit.Assert.*;
+import java.util.stream.Collectors;
+
/**
* Class to tests the JSON schema features
*/
@@ -67,7 +70,7 @@ public class JSONSchemaIT extends BaseIT {
DEFAULT_TRYING_TRIES);
TestUtils.createScope(DUMMY_SCOPE, "Dummy scope", scopeService);
- keepTrying("Scope "+ DUMMY_SCOPE +" not found in the required time", () -> scopeService.getScope(DUMMY_SCOPE),
+ keepTrying("Scope " + DUMMY_SCOPE + " not found in the required time", () -> scopeService.getScope(DUMMY_SCOPE),
Objects::nonNull, DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES);
}
@@ -242,6 +245,53 @@ public class JSONSchemaIT extends BaseIT {
DEFAULT_TRYING_TRIES);
}
+ @Test
+ public void testValidateEvents_valid() throws Exception {
+ assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/1-0-0"));
+ assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/properties/1-0-0"));
+ assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/properties/interests/1-0-0"));
+
+ // Test that at first the flattened event is not valid.
+ assertFalse(schemaService.isEventValid(resourceAsString("schemas/event-flattened-valid.json")));
+
+ // save schemas and check the event pass the validation
+ schemaService.saveSchema(resourceAsString("schemas/schema-flattened.json"));
+ schemaService.saveSchema(resourceAsString("schemas/schema-flattened-flattenedProperties.json"));
+ schemaService.saveSchema(resourceAsString("schemas/schema-flattened-flattenedProperties-interests.json"));
+ schemaService.saveSchema(resourceAsString("schemas/schema-flattened-properties.json"));
+
+ StringBuilder listEvents = new StringBuilder("");
+ listEvents
+ .append("[")
+ .append(resourceAsString("schemas/event-flattened-valid.json"))
+ .append("]");
+
+ keepTrying("No error should have been detected",
+ () -> {
+ try {
+ return schemaService.validateEvents(listEvents.toString()).isEmpty();
+ } catch (Exception e) {
+ return false;
+ }
+ },
+ isValid -> isValid, DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES);
+
+ StringBuilder listInvalidEvents = new StringBuilder("");
+ listInvalidEvents
+ .append("[")
+ .append(resourceAsString("schemas/event-flattened-invalid-1.json")).append(",")
+ .append(resourceAsString("schemas/event-flattened-invalid-2.json")).append(",")
+ .append(resourceAsString("schemas/event-flattened-invalid-3.json")).append(",")
+ .append(resourceAsString("schemas/event-flattened-invalid-3.json"))
+ .append("]");
+ Map<String, Set<ValidationError>> errors = schemaService.validateEvents(listInvalidEvents.toString());
+
+ assertEquals(9, errors.get("flattened").size());
+ // Verify that error on interests.football appear only once even if two events have the issue
+ assertEquals(1, errors.get("flattened").stream().filter(validationError -> validationError.getError().startsWith("$.flattenedProperties.interests.football")).collect(Collectors.toList()).size());
+ }
+
+
@Test
public void testFlattenedProperties() throws Exception {
assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/1-0-0"));
@@ -269,15 +319,15 @@ public class JSONSchemaIT extends BaseIT {
// check that range query is not working on flattened props:
Condition condition = new Condition(definitionsService.getConditionType("eventPropertyCondition"));
- condition.setParameter("propertyName","flattenedProperties.interests.cars");
- condition.setParameter("comparisonOperator","greaterThan");
+ condition.setParameter("propertyName", "flattenedProperties.interests.cars");
+ condition.setParameter("comparisonOperator", "greaterThan");
condition.setParameter("propertyValueInteger", 2);
assertNull(persistenceService.query(condition, null, Event.class, 0, -1));
// check that term query is working on flattened props:
condition = new Condition(definitionsService.getConditionType("eventPropertyCondition"));
- condition.setParameter("propertyName","flattenedProperties.interests.cars");
- condition.setParameter("comparisonOperator","equals");
+ condition.setParameter("propertyName", "flattenedProperties.interests.cars");
+ condition.setParameter("comparisonOperator", "equals");
condition.setParameter("propertyValueInteger", 15);
List<Event> events = persistenceService.query(condition, null, Event.class, 0, -1).getList();
assertEquals(1, events.size());
@@ -325,8 +375,8 @@ public class JSONSchemaIT extends BaseIT {
// wait for the event to be indexed
Condition condition = new Condition(definitionsService.getConditionType("eventPropertyCondition"));
- condition.setParameter("propertyName","properties.marker.keyword");
- condition.setParameter("comparisonOperator","equals");
+ condition.setParameter("propertyName", "properties.marker.keyword");
+ condition.setParameter("comparisonOperator", "equals");
condition.setParameter("propertyValue", eventMarker);
List<Event> events = keepTrying("The event should have been persisted",
() -> persistenceService.query(condition, null, Event.class), results -> results.size() == 1,
diff --git a/manual/src/main/asciidoc/jsonSchema/json-schema-develop.adoc b/manual/src/main/asciidoc/jsonSchema/json-schema-develop.adoc
index 83bb9dc5b..45ec0d50a 100644
--- a/manual/src/main/asciidoc/jsonSchema/json-schema-develop.adoc
+++ b/manual/src/main/asciidoc/jsonSchema/json-schema-develop.adoc
@@ -78,3 +78,63 @@ towards the incorrect property:
}
]
----
+
+==== validateEvents endpoint
+
+A dedicated Admin endpoint (requires authentication), accessible at: `cxs/jsonSchema/validateEvents`, was created to validate a list of event at once against JSON Schemas loaded in Apache Unomi.
+
+For example, sending a list of event not matching a schema:
+[source]
+----
+curl --request POST \
+ --url http://localhost:8181/cxs/jsonSchema/validateEvents \
+ --user karaf:karaf \
+ --header 'Content-Type: application/json' \
+ --data '[{
+ "eventType": "view",
+ "scope": "scope",
+ "properties": {
+ "workspace": "no_workspace",
+ "path": "some/path",
+ "unknowProperty": "not valid"
+ }, {
+ "eventType": "view",
+ "scope": "scope",
+ "properties": {
+ "workspace": "no_workspace",
+ "path": "some/path",
+ "unknowProperty": "not valid",
+ "secondUnknowProperty": "also not valid"
+ }, {
+ "eventType": "notKnownEvent",
+ "scope": "scope",
+ "properties": {
+ "workspace": "no_workspace",
+ "path": "some/path"
+ }
+}]'
+----
+
+Would return the errors grouped by event type as the following:
+
+[source]
+----
+{
+ "view": [
+ {
+ "error": "There are unevaluated properties at following paths $.properties.unknowProperty"
+ },
+ {
+ "error": "There are unevaluated properties at following paths $.properties.secondUnknowProperty"
+ }
+ ],
+ "notKnownEvent": [
+ {
+ "error": "No Schema found for this event type"
+ }
+ ]
+}
+----
+
+If several events have the same issue, only one message is returned for this issue.
+