You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2016/03/23 11:06:40 UTC
lucene-solr:apiv2: SOLR-8029: Validate Json spec
Repository: lucene-solr
Updated Branches:
refs/heads/apiv2 c7f58b820 -> 38d542226
SOLR-8029: Validate Json spec
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/38d54222
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/38d54222
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/38d54222
Branch: refs/heads/apiv2
Commit: 38d542226cbbdda1d0b7b48323f3c235a300e92f
Parents: c7f58b8
Author: Noble Paul <no...@apache.org>
Authored: Wed Mar 23 15:36:20 2016 +0530
Committer: Noble Paul <no...@apache.org>
Committed: Wed Mar 23 15:36:20 2016 +0530
----------------------------------------------------------------------
.../apache/solr/util/JsonSchemaValidator.java | 243 ++++++++++++++-----
.../resources/apispec/collections.Commands.json | 3 +-
.../apispec/core.config.Params.Commands.json | 1 -
.../org/apache/solr/util/JsonValidatorTest.java | 34 ++-
4 files changed, 214 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38d54222/solr/core/src/java/org/apache/solr/util/JsonSchemaValidator.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/JsonSchemaValidator.java b/solr/core/src/java/org/apache/solr/util/JsonSchemaValidator.java
index 723f1840..1118fa8 100644
--- a/solr/core/src/java/org/apache/solr/util/JsonSchemaValidator.java
+++ b/solr/core/src/java/org/apache/solr/util/JsonSchemaValidator.java
@@ -18,6 +18,8 @@ package org.apache.solr.util;
*/
+import java.util.Collections;
+import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
@@ -31,59 +33,168 @@ import static java.util.Collections.unmodifiableMap;
import static java.util.function.Function.identity;
import static java.util.stream.Collectors.toMap;
-public class JsonSchemaValidator {
- private final Map jsonSchema;
-
+/**A very basic json schema parsing and data validation tool.
+ */
+public class JsonSchemaValidator {
+ private final Attribute root;
public JsonSchemaValidator(Map jsonSchema) {
- this.jsonSchema = jsonSchema;
+ root = new Attribute(null);
+ root.isRequired = true;
List<String> errs = new LinkedList<>();
- validateObjectDef(jsonSchema, errs);
+ root.validateSchema(jsonSchema, errs);
if(!errs.isEmpty()){
throw new RuntimeException("Invalid schema. "+ StrUtils.join(errs,'|'));
}
}
- private void validateObjectDef(Map jsonSchema, List<String> errs) {
- for (ObjectAttribute attr : ObjectAttribute.values()) {
- attr.validate(jsonSchema, errs);
+ private static class Attribute {
+ final Attribute parent;
+ Type type;
+ Type arrayElementType;
+ boolean isRequired = false;
+ Boolean additionalProperties;
+ Object validateData;
+ Map<String, Attribute> children;
+
+ private Attribute(Attribute parent) {
+ this.parent = parent;
}
- jsonSchema.keySet().forEach(o -> {
- if (!knownAttributes.containsKey(o)) errs.add("Unknown key : " + o);
- });
- Map m = (Map) jsonSchema.get("properties");
- if(m != null){
- for (Object o : m.entrySet()) {
- Map.Entry e = (Map.Entry) o;
- if (e.getValue() instanceof Map) {
- Map od = (Map) e.getValue();
- validateObjectDef(od,errs);
+
+ private void validateSchema(Map jsonSchema, List<String> errs) {
+ for (ObjectAttribute attr : ObjectAttribute.values()) {
+ attr.validate(jsonSchema, this, errs);
+ }
+ jsonSchema.keySet().forEach(o -> {
+ if (!knownAttributes.containsKey(o)) errs.add("Unknown key : " + o);
+ });
+ if (!errs.isEmpty()) return;
+ Type type = Type.get(jsonSchema.get("type"));
+ if (type == Type.OBJECT) {
+ Map m = (Map) jsonSchema.get("properties");
+ if (m != null) {
+ for (Object o : m.entrySet()) {
+ Map.Entry e = (Map.Entry) o;
+ if (e.getValue() instanceof Map) {
+ Map od = (Map) e.getValue();
+ if (children == null) children = new LinkedHashMap<>();
+ Attribute child = new Attribute(this);
+ children.put((String) e.getKey(), child);
+ child.validateSchema(od, errs);
+ } else {
+ errs.add("Invalid Object definition for field " + e.getKey());
+ }
+ }
} else {
- errs.add("Invalid Object definition for field " +e.getKey());
+ additionalProperties = Boolean.TRUE;
+ }
+ }
+ for (ObjectAttribute attr : ObjectAttribute.values()) {
+ attr.postValidateSchema(jsonSchema, this, errs);
+ }
+
+ }
+
+ private void validate(String key, Object data, List<String> errs) {
+ if (data == null) {
+ if (isRequired) {
+ errs.add("Missing field '" + key+"'");
+ return;
+ }
+ } else {
+ type.valdateData (key, data, this,errs);
+ if(!errs.isEmpty()) return;
+ if (children != null && type == Type.OBJECT) {
+ for (Map.Entry<String, Attribute> e : children.entrySet()) {
+ e.getValue().validate(e.getKey(), ((Map) data).get(e.getKey()), errs);
+ }
+ if (Boolean.TRUE != additionalProperties) {
+ for (Object o : ((Map) data).keySet()) {
+ if (!children.containsKey(o)) {
+ errs.add("Unknown field '" + o + "' in object : " + Utils.toJSONString(data));
+ }
+ }
+ }
}
}
}
}
- public List<String> validateJson(Map json) {
- return null;
+ public List<String> validateJson(Object data) {
+ List<String> errs = new LinkedList<>();
+ root.validate(null, data, errs);
+ return errs.isEmpty() ? null : errs;
}
enum ObjectAttribute {
- type(true, Type.STRING),
- properties(false, Type.OBJECT),
+ type(true, Type.STRING) {
+ @Override
+ public void validate(Map attrSchema, Attribute attr, List<String> errors) {
+ super.validate(attrSchema, attr, errors);
+ attr.type = Type.get(attrSchema.get(key));
+ }
+ },
+ properties(false, Type.OBJECT) {
+ @Override
+ public void validate(Map attrSchema, Attribute attr, List<String> errors) {
+ super.validate(attrSchema, attr, errors);
+ if (attr.type != Type.OBJECT) return;
+ Object val = attrSchema.get(key);
+ if (val == null) {
+ Object additional = attrSchema.get(additionalProperties.key);
+ if (!Boolean.TRUE.equals(additional)) {
+ errors.add("'properties' tag is missing, additionalProperties=true is expected" + Utils.toJSONString(attrSchema));
+ }
+ }
+ }
+ },
additionalProperties(false, Type.BOOLEAN),
- required(false, Type.ARRAY),
- items(false,Type.OBJECT),
+ items(false, Type.OBJECT) {
+ @Override
+ public void validate(Map attrSchema, Attribute attr, List<String> errors) {
+ super.validate(attrSchema, attr, errors);
+ Object itemsVal = attrSchema.get(key);
+ if (itemsVal != null) {
+ if (attr.type != Type.ARRAY) {
+ errors.add("Only 'array' can have 'items'");
+ return;
+ } else {
+ if (itemsVal instanceof Map) {
+ Map val = (Map) itemsVal;
+ Object value = val.get(type.key);
+ Type t = Type.get(String.valueOf(value));
+ if (t == null) {
+ errors.add("Unknown array type " + Utils.toJSONString(attrSchema));
+ } else {
+ attr.arrayElementType = t;
+ }
+ }
+ }
+ }
+ }
+ },
__default(false,Type.UNKNOWN),
- description(false, Type.ARRAY),
+ description(false, Type.STRING),
documentation(false, Type.STRING),
oneOf(false, Type.ARRAY),
id(false, Type.STRING),
_ref(false, Type.STRING),
- _schema(false, Type.STRING);
-
+ _schema(false, Type.STRING),
+ required(false, Type.ARRAY) {
+ @Override
+ public void postValidateSchema(Map attrSchema, Attribute attr, List<String> errors) {
+ Object val = attrSchema.get(key);
+ if (val instanceof List) {
+ List list = (List) val;
+ if (attr.children != null) {
+ for (Map.Entry<String, Attribute> e : attr.children.entrySet()) {
+ if (list.contains(e.getKey())) e.getValue().isRequired = true;
+ }
+ }
+ }
+ }
+ };
final String key;
final boolean _required;
@@ -93,15 +204,19 @@ public class JsonSchemaValidator {
return key;
}
- public void validate(Map attributeDefinition, List<String> errors) {
- Object val = attributeDefinition.get(key);
+ void validate(Map attrSchema, Attribute attr, List<String> errors) {
+ Object val = attrSchema.get(key);
if (val == null) {
- if (_required) errors.add("Missing required attribute '" + key+ "' in object "+ Utils.toJSONString(attributeDefinition) );
+ if (_required)
+ errors.add("Missing required attribute '" + key + "' in object " + Utils.toJSONString(attrSchema));
} else {
if (!typ.validate(val)) errors.add(key + " should be of type " + typ._name);
}
}
+ void postValidateSchema(Map attrSchema, Attribute attr, List<String> errs) {
+ }
+
ObjectAttribute(boolean required, Type type) {
this.key = name().replaceAll("__","").replace('_', '$');
this._required = required;
@@ -110,49 +225,51 @@ public class JsonSchemaValidator {
}
enum Type {
- STRING {
+ STRING(o -> o instanceof String),
+ ARRAY(o -> o instanceof List) {
@Override
- boolean validate(Object o) {
- return o instanceof String;
- }
- },
- ARRAY {
- @Override
- boolean validate(Object o) {
- return o instanceof List || o instanceof String;
+ public void valdateData(String key, Object o, Attribute attr, List<String> errs) {
+ List l = o instanceof List ? (List) o : Collections.singletonList(o);
+ if (attr.arrayElementType != null) {
+ for (Object elem : l) {
+ if (!attr.arrayElementType.validate(elem)) {
+ errs.add("Expected elements of type : " + key + " but found : " + Utils.toJSONString(o));
+ break;
+ }
+ }
+ }
}
},
- NUMBER {
- @Override
- boolean validate(Object o) {
- return o instanceof Number;
- }
- }, BOOLEAN {
- @Override
- boolean validate(Object o) {
- return o instanceof Boolean;
- }
- }, OBJECT {
- @Override
- boolean validate(Object o) {
- return o instanceof Map;
- }
- }, UNKNOWN {
- @Override
- boolean validate(Object o) {
- return true;
- }
- };
+ NUMBER(o -> o instanceof Number),
+ BOOLEAN(o -> o instanceof Boolean),
+ OBJECT(o -> o instanceof Map),
+ UNKNOWN((o -> true));
final String _name;
- Type() {
+ final java.util.function.Predicate typeValidator;
+
+ Type(java.util.function.Predicate validator) {
_name = this.name().toLowerCase(Locale.ROOT);
+ this.typeValidator = validator;
}
- abstract boolean validate(Object o);
+ boolean validate(Object o) {
+ return typeValidator.test(o);
+ }
+
+ void valdateData(String key, Object o, Attribute attr, List<String> errs) {
+ if (!typeValidator.test(o)) errs.add("Expected type : " + _name + " but found : " + Utils.toJSONString(o));
+ }
+ static Type get(Object type) {
+ for (Type t : Type.values()) {
+ if (t._name.equals(type)) return t;
+ }
+ return null;
+ }
}
+
static final Map<String, ObjectAttribute> knownAttributes = unmodifiableMap(asList(ObjectAttribute.values()).stream().collect(toMap(ObjectAttribute::getKey, identity())));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38d54222/solr/core/src/resources/apispec/collections.Commands.json
----------------------------------------------------------------------
diff --git a/solr/core/src/resources/apispec/collections.Commands.json b/solr/core/src/resources/apispec/collections.Commands.json
index 425ee96..753e961 100644
--- a/solr/core/src/resources/apispec/collections.Commands.json
+++ b/solr/core/src/resources/apispec/collections.Commands.json
@@ -40,7 +40,7 @@
}
},
"numShards": {
- "type": "string",
+ "type": "number",
"description": "The number of shards to be created as part of the collection. This is a required parameter when using the 'compositeId' router."
},
"shards": {
@@ -65,7 +65,6 @@
"type": "string"
},
"description":"replica placement rules. See the section Rule-based Replica Placement for details."
-
},
"snitch": {
"type":"array",
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38d54222/solr/core/src/resources/apispec/core.config.Params.Commands.json
----------------------------------------------------------------------
diff --git a/solr/core/src/resources/apispec/core.config.Params.Commands.json b/solr/core/src/resources/apispec/core.config.Params.Commands.json
index a3b054c..597026d 100644
--- a/solr/core/src/resources/apispec/core.config.Params.Commands.json
+++ b/solr/core/src/resources/apispec/core.config.Params.Commands.json
@@ -12,7 +12,6 @@
"set:": {
"type":"object",
"description":"add or overwrite one or more param sets",
- "type": "object",
"additionalProperties": true
},
"unset": {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38d54222/solr/core/src/test/org/apache/solr/util/JsonValidatorTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/JsonValidatorTest.java b/solr/core/src/test/org/apache/solr/util/JsonValidatorTest.java
index cef319f..3b57f06 100644
--- a/solr/core/src/test/org/apache/solr/util/JsonValidatorTest.java
+++ b/solr/core/src/test/org/apache/solr/util/JsonValidatorTest.java
@@ -17,11 +17,17 @@ package org.apache.solr.util;
* limitations under the License.
*/
+import java.util.List;
import java.util.Map;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.api.ApiBag;
import org.apache.solr.common.util.Map2;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+
+import static org.apache.solr.common.util.Map2.NOT_NULL;
+import static org.apache.solr.common.util.Utils.toJSONString;
public class JsonValidatorTest extends SolrTestCaseJ4 {
@@ -39,13 +45,39 @@ public class JsonValidatorTest extends SolrTestCaseJ4 {
checkSchema("core.SchemaEdit");
}
+
+
+ public void testSchemaValidation() {
+ Map2 spec = ApiBag.getSpec("collections.commands").getSpec();
+ Map createSchema = spec.getMap("commands", NOT_NULL).getMap("create-alias", NOT_NULL);
+ JsonSchemaValidator validator = new JsonSchemaValidator(createSchema);
+ List<String> errs = validator.validateJson(Utils.fromJSONString("{name : x, collections: [ c1 , c2]}"));
+ assertNull(toJSONString(errs), errs);
+ errs = validator.validateJson(Utils.fromJSONString("{name : x, collections: c1 }"));
+ assertNull(toJSONString(errs), errs);
+ errs = validator.validateJson(Utils.fromJSONString("{name : x, x:y, collections: [ c1 , c2]}"));
+ assertFalse(toJSONString(errs), errs.isEmpty());
+ assertTrue(toJSONString(errs), errs.get(0).contains("Unknown"));
+ errs = validator.validateJson(Utils.fromJSONString("{name : 123, collections: c1 }"));
+ assertFalse(toJSONString(errs), errs.isEmpty());
+ assertTrue(toJSONString(errs), errs.get(0).contains("Expected type"));
+ errs = validator.validateJson(Utils.fromJSONString("{x:y, collections: [ c1 , c2]}"));
+ assertEquals(toJSONString(errs),2, errs.size());
+ assertTrue(toJSONString(errs), StrUtils.join(errs, '|').contains("Missing field"));
+ assertTrue(toJSONString(errs), StrUtils.join(errs, '|').contains("Unknown"));
+ errs = validator.validateJson(Utils.fromJSONString("{name : x, collections: [ 1 , 2]}"));
+ assertFalse(toJSONString(errs), errs.isEmpty());
+ assertTrue(toJSONString(errs), errs.get(0).contains("Expected elements of type"));
+
+ }
+
private void checkSchema(String name) {
Map2 spec = ApiBag.getSpec(name).getSpec();
Map commands = (Map) spec.get("commands");
for (Object o : commands.entrySet()) {
Map.Entry cmd = (Map.Entry) o;
try {
- new JsonSchemaValidator((Map) cmd.getValue());
+ JsonSchemaValidator validator = new JsonSchemaValidator((Map) cmd.getValue());
} catch (Exception e) {
throw new RuntimeException("Error in command "+ cmd.getKey() +" in schema "+name, e);
}