You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@avro.apache.org by "github-advanced-security[bot] (via GitHub)" <gi...@apache.org> on 2023/12/19 12:01:54 UTC
Re: [PR] AVRO-3666 [Java] Use the new schema parser [avro]
github-advanced-security[bot] commented on code in PR #2642:
URL: https://github.com/apache/avro/pull/2642#discussion_r1431314277
##########
lang/java/avro/src/main/java/org/apache/avro/JsonSchemaParser.java:
##########
@@ -57,26 +57,34 @@
for (String fragment : fragments) {
buffer.append(fragment);
}
- return new JsonSchemaParser().parse(new ParseContext(NameValidator.NO_VALIDATION), buffer, null);
+
+ boolean saved = Schema.getValidateDefaults();
+ try {
+ ParseContext context = new ParseContext(NameValidator.NO_VALIDATION);
+
+ Schema schema = new JsonSchemaParser().parse(context, buffer, true);
+ context.commit();
+ Schema.setValidateDefaults(false);
Review Comment:
## Deprecated method or constructor invocation
Invoking [Schema.setValidateDefaults](1) should be avoided because it has been deprecated.
[Show more details](https://github.com/apache/avro/security/code-scanning/3148)
##########
lang/java/avro/src/main/java/org/apache/avro/Schema.java:
##########
@@ -1867,47 +1782,70 @@
}
}
- /**
- * Parse named schema in order to fill names map. This method does not parse
- * field of record/error schema.
- *
- * @param schema : json schema representation.
- * @param names : map of named schema.
- * @param currentNameSpace : current working name space.
- * @return schema.
- */
- static Schema parseNamesDeclared(JsonNode schema, Names names, String currentNameSpace) {
+ /** @see #parse(String) */
+ static Schema parse(JsonNode schema, ParseContext context, String currentNameSpace) {
if (schema == null) {
- return null;
+ throw new SchemaParseException("Cannot parse <null> schema");
}
- if (schema.isObject()) {
-
- String type = Schema.getOptionalText(schema, "type");
+ if (schema.isTextual()) { // name
+ return context.find(schema.textValue(), currentNameSpace);
+ } else if (schema.isObject()) {
+ Schema result;
+ String type = getRequiredText(schema, "type", "No type");
Name name = null;
-
+ String space = null;
String doc = null;
- Schema result = null;
final boolean isTypeError = "error".equals(type);
final boolean isTypeRecord = "record".equals(type);
final boolean isTypeEnum = "enum".equals(type);
final boolean isTypeFixed = "fixed".equals(type);
-
if (isTypeRecord || isTypeError || isTypeEnum || isTypeFixed) {
- String space = getOptionalText(schema, "namespace");
+ space = getOptionalText(schema, "namespace");
doc = getOptionalText(schema, "doc");
if (space == null)
space = currentNameSpace;
name = new Name(getRequiredText(schema, "name", "No name in schema"), space);
}
- if (isTypeRecord || isTypeError) { // record
+ if (PRIMITIVES.containsKey(type)) { // primitive
+ result = create(PRIMITIVES.get(type));
+ } else if (isTypeRecord || isTypeError) { // record
+ List<Field> fields = new ArrayList<>();
result = new RecordSchema(name, doc, isTypeError);
- names.add(result);
+ context.put(result);
JsonNode fieldsNode = schema.get("fields");
-
if (fieldsNode == null || !fieldsNode.isArray())
throw new SchemaParseException("Record has no fields: " + schema);
- exploreFields(fieldsNode, names, name != null ? name.space : null);
-
+ for (JsonNode field : fieldsNode) {
+ String fieldName = getRequiredText(field, "name", "No field name");
+ String fieldDoc = getOptionalText(field, "doc");
+ JsonNode fieldTypeNode = field.get("type");
+ if (fieldTypeNode == null)
+ throw new SchemaParseException("No field type: " + field);
+ Schema fieldSchema = parse(fieldTypeNode, context, name.space);
+ Field.Order order = Field.Order.ASCENDING;
+ JsonNode orderNode = field.get("order");
+ if (orderNode != null)
+ order = Field.Order.valueOf(orderNode.textValue().toUpperCase(Locale.ENGLISH));
+ JsonNode defaultValue = field.get("default");
+ if (defaultValue != null
+ && (Type.FLOAT.equals(fieldSchema.getType()) || Type.DOUBLE.equals(fieldSchema.getType()))
+ && defaultValue.isTextual())
+ defaultValue = new DoubleNode(Double.parseDouble(defaultValue.textValue()));
Review Comment:
## Missing catch of NumberFormatException
Potential uncaught 'java.lang.NumberFormatException'.
[Show more details](https://github.com/apache/avro/security/code-scanning/2965)
##########
lang/java/avro/src/main/java/org/apache/avro/ParseContext.java:
##########
@@ -220,6 +232,77 @@
newSchemas.clear();
}
+ /**
+ * Resolve all (named) schemas that were parsed. This resolves all forward
+ * references, even if parsed from different files.
+ *
+ * @return all parsed schemas, in the order they were parsed
+ * @throws AvroTypeException if a reference cannot be resolved
+ */
+ public List<Schema> resolveAllTypes() {
+ if (hasNewSchemas()) {
+ throw new IllegalStateException("Types cannot be resolved unless the ParseContext is committed.");
+ }
+
+ if (!isResolved) {
+ NameValidator saved = Schema.getNameValidator();
+ try {
+ Schema.setNameValidator(nameValidator); // Ensure we use the same validation.
+ HashMap<String, Schema> result = new LinkedHashMap<>(oldSchemas);
+ SchemaResolver.ResolvingVisitor visitor = new SchemaResolver.ResolvingVisitor(null, result::get, false);
+ Function<Schema, Schema> resolver = schema -> Schemas.visit(schema, visitor.withRoot(schema));
+ for (Map.Entry<String, Schema> entry : result.entrySet()) {
+ entry.setValue(resolver.apply(entry.getValue()));
+ }
+ oldSchemas.putAll(result);
+ isResolved = true;
+ } finally {
+ Schema.setNameValidator(saved);
+ }
+ }
+
+ return new ArrayList<>(oldSchemas.values());
+ }
+
+ /**
+ * Try to resolve unresolved references in a schema using the types known to
+ * this context. It is advisable to call {@link #resolveAllTypes()} first if you
+ * want the returned types to be stable.
+ *
+ * @param schema the schema resolve
+ * @return the fully resolved schema if possible, {@code null} otherwise
+ */
+ public Schema tryResolve(Schema schema) {
+ if (schema == null) {
+ return null;
+ }
+ return resolve(schema, true);
+ }
+
+ /**
+ * Resolve unresolved references in a schema using the types known to this
+ * context. It is advisable to call {@link #resolveAllTypes()} first if you want
+ * the returned types to be stable.
+ *
+ * @param schema the schema resolve
+ * @return the fully resolved schema
+ * @throws AvroTypeException if the schema cannot be resolved
+ */
+ public Schema resolve(Schema schema) {
+ return resolve(schema, false);
+ }
+
+ public Schema resolve(Schema schema, boolean returnNullUponFailure) {
+ NameValidator saved = Schema.getNameValidator();
+ try {
+ Schema.setNameValidator(nameValidator); // Ensure we use the same validation.
Review Comment:
## Deprecated method or constructor invocation
Invoking [Schema.setNameValidator](1) should be avoided because it has been deprecated.
[Show more details](https://github.com/apache/avro/security/code-scanning/3154)
##########
lang/java/avro/src/main/java/org/apache/avro/Schema.java:
##########
@@ -1867,47 +1782,70 @@
}
}
- /**
- * Parse named schema in order to fill names map. This method does not parse
- * field of record/error schema.
- *
- * @param schema : json schema representation.
- * @param names : map of named schema.
- * @param currentNameSpace : current working name space.
- * @return schema.
- */
- static Schema parseNamesDeclared(JsonNode schema, Names names, String currentNameSpace) {
+ /** @see #parse(String) */
+ static Schema parse(JsonNode schema, ParseContext context, String currentNameSpace) {
if (schema == null) {
- return null;
+ throw new SchemaParseException("Cannot parse <null> schema");
}
- if (schema.isObject()) {
-
- String type = Schema.getOptionalText(schema, "type");
+ if (schema.isTextual()) { // name
+ return context.find(schema.textValue(), currentNameSpace);
+ } else if (schema.isObject()) {
+ Schema result;
+ String type = getRequiredText(schema, "type", "No type");
Name name = null;
-
+ String space = null;
String doc = null;
- Schema result = null;
final boolean isTypeError = "error".equals(type);
final boolean isTypeRecord = "record".equals(type);
final boolean isTypeEnum = "enum".equals(type);
final boolean isTypeFixed = "fixed".equals(type);
-
if (isTypeRecord || isTypeError || isTypeEnum || isTypeFixed) {
- String space = getOptionalText(schema, "namespace");
+ space = getOptionalText(schema, "namespace");
doc = getOptionalText(schema, "doc");
if (space == null)
space = currentNameSpace;
name = new Name(getRequiredText(schema, "name", "No name in schema"), space);
}
- if (isTypeRecord || isTypeError) { // record
+ if (PRIMITIVES.containsKey(type)) { // primitive
+ result = create(PRIMITIVES.get(type));
+ } else if (isTypeRecord || isTypeError) { // record
+ List<Field> fields = new ArrayList<>();
result = new RecordSchema(name, doc, isTypeError);
- names.add(result);
+ context.put(result);
JsonNode fieldsNode = schema.get("fields");
-
if (fieldsNode == null || !fieldsNode.isArray())
throw new SchemaParseException("Record has no fields: " + schema);
- exploreFields(fieldsNode, names, name != null ? name.space : null);
-
+ for (JsonNode field : fieldsNode) {
+ String fieldName = getRequiredText(field, "name", "No field name");
+ String fieldDoc = getOptionalText(field, "doc");
+ JsonNode fieldTypeNode = field.get("type");
+ if (fieldTypeNode == null)
+ throw new SchemaParseException("No field type: " + field);
+ Schema fieldSchema = parse(fieldTypeNode, context, name.space);
Review Comment:
## Dereferenced variable may be null
Variable [name](1) may be null at this access because of [this](2) assignment.
[Show more details](https://github.com/apache/avro/security/code-scanning/3157)
##########
lang/java/avro/src/main/java/org/apache/avro/JsonSchemaParser.java:
##########
@@ -57,26 +57,34 @@
for (String fragment : fragments) {
buffer.append(fragment);
}
- return new JsonSchemaParser().parse(new ParseContext(NameValidator.NO_VALIDATION), buffer, null);
+
+ boolean saved = Schema.getValidateDefaults();
Review Comment:
## Deprecated method or constructor invocation
Invoking [Schema.getValidateDefaults](1) should be avoided because it has been deprecated.
[Show more details](https://github.com/apache/avro/security/code-scanning/3147)
##########
lang/java/avro/src/main/java/org/apache/avro/ParseContext.java:
##########
@@ -220,6 +232,77 @@
newSchemas.clear();
}
+ /**
+ * Resolve all (named) schemas that were parsed. This resolves all forward
+ * references, even if parsed from different files.
+ *
+ * @return all parsed schemas, in the order they were parsed
+ * @throws AvroTypeException if a reference cannot be resolved
+ */
+ public List<Schema> resolveAllTypes() {
+ if (hasNewSchemas()) {
+ throw new IllegalStateException("Types cannot be resolved unless the ParseContext is committed.");
+ }
+
+ if (!isResolved) {
+ NameValidator saved = Schema.getNameValidator();
Review Comment:
## Deprecated method or constructor invocation
Invoking [Schema.getNameValidator](1) should be avoided because it has been deprecated.
[Show more details](https://github.com/apache/avro/security/code-scanning/3150)
##########
lang/java/avro/src/main/java/org/apache/avro/ParseContext.java:
##########
@@ -220,6 +232,77 @@
newSchemas.clear();
}
+ /**
+ * Resolve all (named) schemas that were parsed. This resolves all forward
+ * references, even if parsed from different files.
+ *
+ * @return all parsed schemas, in the order they were parsed
+ * @throws AvroTypeException if a reference cannot be resolved
+ */
+ public List<Schema> resolveAllTypes() {
+ if (hasNewSchemas()) {
+ throw new IllegalStateException("Types cannot be resolved unless the ParseContext is committed.");
+ }
+
+ if (!isResolved) {
+ NameValidator saved = Schema.getNameValidator();
+ try {
+ Schema.setNameValidator(nameValidator); // Ensure we use the same validation.
+ HashMap<String, Schema> result = new LinkedHashMap<>(oldSchemas);
+ SchemaResolver.ResolvingVisitor visitor = new SchemaResolver.ResolvingVisitor(null, result::get, false);
+ Function<Schema, Schema> resolver = schema -> Schemas.visit(schema, visitor.withRoot(schema));
+ for (Map.Entry<String, Schema> entry : result.entrySet()) {
+ entry.setValue(resolver.apply(entry.getValue()));
+ }
+ oldSchemas.putAll(result);
+ isResolved = true;
+ } finally {
+ Schema.setNameValidator(saved);
Review Comment:
## Deprecated method or constructor invocation
Invoking [Schema.setNameValidator](1) should be avoided because it has been deprecated.
[Show more details](https://github.com/apache/avro/security/code-scanning/3152)
##########
lang/java/avro/src/main/java/org/apache/avro/JsonSchemaParser.java:
##########
@@ -57,26 +57,34 @@
for (String fragment : fragments) {
buffer.append(fragment);
}
- return new JsonSchemaParser().parse(new ParseContext(NameValidator.NO_VALIDATION), buffer, null);
+
+ boolean saved = Schema.getValidateDefaults();
+ try {
+ ParseContext context = new ParseContext(NameValidator.NO_VALIDATION);
+
+ Schema schema = new JsonSchemaParser().parse(context, buffer, true);
+ context.commit();
+ Schema.setValidateDefaults(false);
+ context.resolveAllTypes();
+ return context.resolve(schema);
+ } finally {
+ // Unless explicitly disabled when needed, defaults should always be validated.
+ Schema.setValidateDefaults(saved);
Review Comment:
## Deprecated method or constructor invocation
Invoking [Schema.setValidateDefaults](1) should be avoided because it has been deprecated.
[Show more details](https://github.com/apache/avro/security/code-scanning/3149)
##########
lang/java/avro/src/main/java/org/apache/avro/ParseContext.java:
##########
@@ -220,6 +232,77 @@
newSchemas.clear();
}
+ /**
+ * Resolve all (named) schemas that were parsed. This resolves all forward
+ * references, even if parsed from different files.
+ *
+ * @return all parsed schemas, in the order they were parsed
+ * @throws AvroTypeException if a reference cannot be resolved
+ */
+ public List<Schema> resolveAllTypes() {
+ if (hasNewSchemas()) {
+ throw new IllegalStateException("Types cannot be resolved unless the ParseContext is committed.");
+ }
+
+ if (!isResolved) {
+ NameValidator saved = Schema.getNameValidator();
+ try {
+ Schema.setNameValidator(nameValidator); // Ensure we use the same validation.
Review Comment:
## Deprecated method or constructor invocation
Invoking [Schema.setNameValidator](1) should be avoided because it has been deprecated.
[Show more details](https://github.com/apache/avro/security/code-scanning/3151)
##########
lang/java/avro/src/main/java/org/apache/avro/ParseContext.java:
##########
@@ -220,6 +232,77 @@
newSchemas.clear();
}
+ /**
+ * Resolve all (named) schemas that were parsed. This resolves all forward
+ * references, even if parsed from different files.
+ *
+ * @return all parsed schemas, in the order they were parsed
+ * @throws AvroTypeException if a reference cannot be resolved
+ */
+ public List<Schema> resolveAllTypes() {
+ if (hasNewSchemas()) {
+ throw new IllegalStateException("Types cannot be resolved unless the ParseContext is committed.");
+ }
+
+ if (!isResolved) {
+ NameValidator saved = Schema.getNameValidator();
+ try {
+ Schema.setNameValidator(nameValidator); // Ensure we use the same validation.
+ HashMap<String, Schema> result = new LinkedHashMap<>(oldSchemas);
+ SchemaResolver.ResolvingVisitor visitor = new SchemaResolver.ResolvingVisitor(null, result::get, false);
+ Function<Schema, Schema> resolver = schema -> Schemas.visit(schema, visitor.withRoot(schema));
+ for (Map.Entry<String, Schema> entry : result.entrySet()) {
+ entry.setValue(resolver.apply(entry.getValue()));
+ }
+ oldSchemas.putAll(result);
+ isResolved = true;
+ } finally {
+ Schema.setNameValidator(saved);
+ }
+ }
+
+ return new ArrayList<>(oldSchemas.values());
+ }
+
+ /**
+ * Try to resolve unresolved references in a schema using the types known to
+ * this context. It is advisable to call {@link #resolveAllTypes()} first if you
+ * want the returned types to be stable.
+ *
+ * @param schema the schema resolve
+ * @return the fully resolved schema if possible, {@code null} otherwise
+ */
+ public Schema tryResolve(Schema schema) {
+ if (schema == null) {
+ return null;
+ }
+ return resolve(schema, true);
+ }
+
+ /**
+ * Resolve unresolved references in a schema using the types known to this
+ * context. It is advisable to call {@link #resolveAllTypes()} first if you want
+ * the returned types to be stable.
+ *
+ * @param schema the schema resolve
+ * @return the fully resolved schema
+ * @throws AvroTypeException if the schema cannot be resolved
+ */
+ public Schema resolve(Schema schema) {
+ return resolve(schema, false);
+ }
+
+ public Schema resolve(Schema schema, boolean returnNullUponFailure) {
+ NameValidator saved = Schema.getNameValidator();
Review Comment:
## Deprecated method or constructor invocation
Invoking [Schema.getNameValidator](1) should be avoided because it has been deprecated.
[Show more details](https://github.com/apache/avro/security/code-scanning/3153)
##########
lang/java/avro/src/main/java/org/apache/avro/Schema.java:
##########
@@ -1488,7 +1399,8 @@
* may refer to it by name.
*/
public static class Parser {
- private Names names = new Names();
+ private final Names names = new Names();
Review Comment:
## Container contents are never accessed
The contents of this container are never accessed.
[Show more details](https://github.com/apache/avro/security/code-scanning/3158)
##########
lang/java/avro/src/main/java/org/apache/avro/ParseContext.java:
##########
@@ -220,6 +232,77 @@
newSchemas.clear();
}
+ /**
+ * Resolve all (named) schemas that were parsed. This resolves all forward
+ * references, even if parsed from different files.
+ *
+ * @return all parsed schemas, in the order they were parsed
+ * @throws AvroTypeException if a reference cannot be resolved
+ */
+ public List<Schema> resolveAllTypes() {
+ if (hasNewSchemas()) {
+ throw new IllegalStateException("Types cannot be resolved unless the ParseContext is committed.");
+ }
+
+ if (!isResolved) {
+ NameValidator saved = Schema.getNameValidator();
+ try {
+ Schema.setNameValidator(nameValidator); // Ensure we use the same validation.
+ HashMap<String, Schema> result = new LinkedHashMap<>(oldSchemas);
+ SchemaResolver.ResolvingVisitor visitor = new SchemaResolver.ResolvingVisitor(null, result::get, false);
+ Function<Schema, Schema> resolver = schema -> Schemas.visit(schema, visitor.withRoot(schema));
+ for (Map.Entry<String, Schema> entry : result.entrySet()) {
+ entry.setValue(resolver.apply(entry.getValue()));
+ }
+ oldSchemas.putAll(result);
+ isResolved = true;
+ } finally {
+ Schema.setNameValidator(saved);
+ }
+ }
+
+ return new ArrayList<>(oldSchemas.values());
+ }
+
+ /**
+ * Try to resolve unresolved references in a schema using the types known to
+ * this context. It is advisable to call {@link #resolveAllTypes()} first if you
+ * want the returned types to be stable.
+ *
+ * @param schema the schema resolve
+ * @return the fully resolved schema if possible, {@code null} otherwise
+ */
+ public Schema tryResolve(Schema schema) {
+ if (schema == null) {
+ return null;
+ }
+ return resolve(schema, true);
+ }
+
+ /**
+ * Resolve unresolved references in a schema using the types known to this
+ * context. It is advisable to call {@link #resolveAllTypes()} first if you want
+ * the returned types to be stable.
+ *
+ * @param schema the schema resolve
+ * @return the fully resolved schema
+ * @throws AvroTypeException if the schema cannot be resolved
+ */
+ public Schema resolve(Schema schema) {
+ return resolve(schema, false);
+ }
+
+ public Schema resolve(Schema schema, boolean returnNullUponFailure) {
+ NameValidator saved = Schema.getNameValidator();
+ try {
+ Schema.setNameValidator(nameValidator); // Ensure we use the same validation.
+ return Schemas.visit(schema,
+ new SchemaResolver.ResolvingVisitor(schema, this::getNamedSchema, returnNullUponFailure));
+ } finally {
+ Schema.setNameValidator(saved);
Review Comment:
## Deprecated method or constructor invocation
Invoking [Schema.setNameValidator](1) should be avoided because it has been deprecated.
[Show more details](https://github.com/apache/avro/security/code-scanning/3155)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@avro.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org