You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by su...@apache.org on 2022/02/16 17:20:13 UTC
[gobblin] branch avro_1_9 updated: Use helper-all v0.2.74 to solve issues around default values. (#3469)
This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch avro_1_9
in repository https://gitbox.apache.org/repos/asf/gobblin.git
The following commit(s) were added to refs/heads/avro_1_9 by this push:
new 3b62702 Use helper-all v0.2.74 to solve issues around default values. (#3469)
3b62702 is described below
commit 3b627022f496616a0d4e7e44fe3ca4fc4f1131e9
Author: Sreeram Ramachandran <sr...@linkedin.com>
AuthorDate: Wed Feb 16 09:20:05 2022 -0800
Use helper-all v0.2.74 to solve issues around default values. (#3469)
The latest version of helper-all fixes the issues seen before w.r.t.
default values, so we can now revert the code and the *.avsc files back
to how they used to be, with two minor exceptions:
1. Check Schema equality using their .toString() representations. Doing
it the old way works for two out of the three instances, but one of
them fails, for reasons I haven't figured out yet.
2. Add a `"default":null` piece to recursive_schema_1_converted.avsc.
This is harmless, and is caused by the fact that the compatibility
helper always adds it if it's a valid default for the schema. See
the comments for FieldBuilder19.setDefault():
https://github.com/linkedin/avro-util/blob/b9e89c55980ea8e5fd3c8d8da362d7195dd2a99c/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/FieldBuilder19.java#L69
To verify that the files are otherwise the same as before:
```
$ for file in gobblin-core-base/src/test/resources/converter/*.avsc; do
> git show 928e0180c471fc4b7a6caee041b001b5b34e1cc6:$file > /tmp/before
> diff <(jq . </tmp/before) <(jq . <$file)
> done
```
---
.../gobblin/converter/filter/AvroSchemaFieldRemoverTest.java | 12 ------------
.../src/test/resources/converter/recursive_schema_1.avsc | 2 +-
.../resources/converter/recursive_schema_1_converted.avsc | 2 +-
.../src/test/resources/converter/recursive_schema_2.avsc | 2 +-
.../resources/converter/recursive_schema_2_converted.avsc | 2 +-
.../converter/recursive_schema_2_not_converted.avsc | 2 +-
gradle/scripts/dependencyDefinitions.gradle | 2 +-
7 files changed, 6 insertions(+), 18 deletions(-)
diff --git a/gobblin-core-base/src/test/java/org/apache/gobblin/converter/filter/AvroSchemaFieldRemoverTest.java b/gobblin-core-base/src/test/java/org/apache/gobblin/converter/filter/AvroSchemaFieldRemoverTest.java
index 76aadbe..1e4d012 100644
--- a/gobblin-core-base/src/test/java/org/apache/gobblin/converter/filter/AvroSchemaFieldRemoverTest.java
+++ b/gobblin-core-base/src/test/java/org/apache/gobblin/converter/filter/AvroSchemaFieldRemoverTest.java
@@ -35,22 +35,10 @@ public class AvroSchemaFieldRemoverTest {
@Test
public void testRemoveFields() throws IllegalArgumentException, IOException {
- // Avro 1.9.2 uses Object as default value and uses internal JacksonUtils.toJson to convert the Object default value
- // to JsonNode. Enum default value is not supported and hence the conversion fails. Updated the input schema and
- // removed the non null default value for enum.
Schema convertedSchema1 = convertSchema("/converter/recursive_schema_1.avsc", "YwchQiH.OjuzrLOtmqLW");
Schema expectedSchema1 = parseSchema("/converter/recursive_schema_1_converted.avsc");
Assert.assertEquals(convertedSchema1.toString(), expectedSchema1.toString());
- // Avro 1.9.2 uses Object as default value and uses internal JacksonUtils.toJson to convert the Object default value
- // to JsonNode. Default value of record type is not supported and hence the conversion fails. Updated the input
- // schema and removed the non null default value for enum. In earlier Avro 1.8.1 there was no such conversion.
- // For example, remove the below default value.
- // "default": {
- // "AUjmPup": null,
- // "IFBRClOa": "zG",
- // "sZIVnwv": null
- // }
Schema convertedSchema2 =
convertSchema("/converter/recursive_schema_2.avsc", "FBuKC.wIINqII.lvaerUEKxBQUWg,eFQjDj.TzuYZajb");
Schema expectedSchema2 = parseSchema("/converter/recursive_schema_2_converted.avsc");
diff --git a/gobblin-core-base/src/test/resources/converter/recursive_schema_1.avsc b/gobblin-core-base/src/test/resources/converter/recursive_schema_1.avsc
index 3b0070a..4b41e2d 100644
--- a/gobblin-core-base/src/test/resources/converter/recursive_schema_1.avsc
+++ b/gobblin-core-base/src/test/resources/converter/recursive_schema_1.avsc
@@ -1 +1 @@
-{"type":"record","name":"VyPswKoukcXEZshQrXnE","namespace":"PKA.POshikUo.flXRgM.aBxSQzgOe","fields":[{"name":"jRqjDF","type":{"type":"record","name":"ZDlOMWcUTCk","namespace":"PKA.POshikUo.flXRgM","fields":[{"name":"GJiZXGQc","type":"int","doc":"Vdj NXTFcrls GsWTlJ Lw WS oWm ZbYt OxeGXFqHeG oWm gQpsyC NXTFcrls GsWTlJ ISn yqw tVlMiJQo XECvKHj JUyZ BGwP PMEQNW yqw qfzNChDuzDx GwBGpC jM BGwP Ox r xOrwhzll lIVyeB","meta":"field_meta"},{"name":"NZUOnQgci","type":["null","string"],"doc":"Vdj N [...]
+{"type":"record","name":"VyPswKoukcXEZshQrXnE","namespace":"PKA.POshikUo.flXRgM.aBxSQzgOe","fields":[{"name":"jRqjDF","type":{"type":"record","name":"ZDlOMWcUTCk","namespace":"PKA.POshikUo.flXRgM","fields":[{"name":"GJiZXGQc","type":"int","doc":"Vdj NXTFcrls GsWTlJ Lw WS oWm ZbYt OxeGXFqHeG oWm gQpsyC NXTFcrls GsWTlJ ISn yqw tVlMiJQo XECvKHj JUyZ BGwP PMEQNW yqw qfzNChDuzDx GwBGpC jM BGwP Ox r xOrwhzll lIVyeB","meta":"field_meta"},{"name":"NZUOnQgci","type":["null","string"],"doc":"Vdj N [...]
diff --git a/gobblin-core-base/src/test/resources/converter/recursive_schema_1_converted.avsc b/gobblin-core-base/src/test/resources/converter/recursive_schema_1_converted.avsc
index 0b4b245..02d2432 100644
--- a/gobblin-core-base/src/test/resources/converter/recursive_schema_1_converted.avsc
+++ b/gobblin-core-base/src/test/resources/converter/recursive_schema_1_converted.avsc
@@ -1 +1 @@
-{"type":"record","name":"VyPswKoukcXEZshQrXnE","namespace":"PKA.POshikUo.flXRgM.aBxSQzgOe","fields":[{"name":"jRqjDF","type":{"type":"record","name":"ZDlOMWcUTCk","namespace":"PKA.POshikUo.flXRgM","fields":[{"name":"GJiZXGQc","type":"int","doc":"Vdj NXTFcrls GsWTlJ Lw WS oWm ZbYt OxeGXFqHeG oWm gQpsyC NXTFcrls GsWTlJ ISn yqw tVlMiJQo XECvKHj JUyZ BGwP PMEQNW yqw qfzNChDuzDx GwBGpC jM BGwP Ox r xOrwhzll lIVyeB","meta":"field_meta"},{"name":"NZUOnQgci","type":["null","string"],"doc":"Vdj N [...]
+{"type":"record","name":"VyPswKoukcXEZshQrXnE","namespace":"PKA.POshikUo.flXRgM.aBxSQzgOe","fields":[{"name":"jRqjDF","type":{"type":"record","name":"ZDlOMWcUTCk","namespace":"PKA.POshikUo.flXRgM","fields":[{"name":"GJiZXGQc","type":"int","doc":"Vdj NXTFcrls GsWTlJ Lw WS oWm ZbYt OxeGXFqHeG oWm gQpsyC NXTFcrls GsWTlJ ISn yqw tVlMiJQo XECvKHj JUyZ BGwP PMEQNW yqw qfzNChDuzDx GwBGpC jM BGwP Ox r xOrwhzll lIVyeB","meta":"field_meta"},{"name":"NZUOnQgci","type":["null","string"],"doc":"Vdj N [...]
diff --git a/gobblin-core-base/src/test/resources/converter/recursive_schema_2.avsc b/gobblin-core-base/src/test/resources/converter/recursive_schema_2.avsc
index c708ecb..5a15eae 100644
--- a/gobblin-core-base/src/test/resources/converter/recursive_schema_2.avsc
+++ b/gobblin-core-base/src/test/resources/converter/recursive_schema_2.avsc
@@ -1 +1 @@
-{"type":"record","name":"XQLFognzBcWX","namespace":"IoV.uIwHpaVy.dUV","fields":[{"name":"eFQjDj","type":["null",{"type":"record","name":"CWlSbaEOPjQ","fields":[{"name":"TzuYZajb","type":"int","doc":"BLD isOtza xj fS jty qGGt PpaPkENfDJ jty HTvfxa"},{"name":"zMMEITWsv","type":["null","string"],"doc":"Pnu fS jty qGGt PpaPkENfDJ jty HTvfxa","default":null},{"name":"kTrp","type":"long","doc":"BLD kTrp fS jty FBuKC"},{"name":"kdYkvr","type":"string","doc":"BLD name fS jty kdYkvr"},{"name":"jf [...]
+{"type":"record","name":"XQLFognzBcWX","namespace":"IoV.uIwHpaVy.dUV","fields":[{"name":"eFQjDj","type":["null",{"type":"record","name":"CWlSbaEOPjQ","fields":[{"name":"TzuYZajb","type":"int","doc":"BLD isOtza xj fS jty qGGt PpaPkENfDJ jty HTvfxa"},{"name":"zMMEITWsv","type":["null","string"],"doc":"Pnu fS jty qGGt PpaPkENfDJ jty HTvfxa","default":null},{"name":"kTrp","type":"long","doc":"BLD kTrp fS jty FBuKC"},{"name":"kdYkvr","type":"string","doc":"BLD name fS jty kdYkvr"},{"name":"jf [...]
diff --git a/gobblin-core-base/src/test/resources/converter/recursive_schema_2_converted.avsc b/gobblin-core-base/src/test/resources/converter/recursive_schema_2_converted.avsc
index 8cff0df..a733019 100644
--- a/gobblin-core-base/src/test/resources/converter/recursive_schema_2_converted.avsc
+++ b/gobblin-core-base/src/test/resources/converter/recursive_schema_2_converted.avsc
@@ -1 +1 @@
-{"type":"record","name":"XQLFognzBcWX","namespace":"IoV.uIwHpaVy.dUV","fields":[{"name":"eFQjDj","type":["null",{"type":"record","name":"CWlSbaEOPjQ","fields":[{"name":"zMMEITWsv","type":["null","string"],"doc":"Pnu fS jty qGGt PpaPkENfDJ jty HTvfxa","default":null},{"name":"kTrp","type":"long","doc":"BLD kTrp fS jty FBuKC"},{"name":"kdYkvr","type":"string","doc":"BLD name fS jty kdYkvr"},{"name":"jfqWhtm","type":"string","doc":"BLD name fS jty jfqWhtm"},{"name":"ORVhoRguhhZ","type":["nu [...]
\ No newline at end of file
+{"type":"record","name":"XQLFognzBcWX","namespace":"IoV.uIwHpaVy.dUV","fields":[{"name":"eFQjDj","type":["null",{"type":"record","name":"CWlSbaEOPjQ","fields":[{"name":"zMMEITWsv","type":["null","string"],"doc":"Pnu fS jty qGGt PpaPkENfDJ jty HTvfxa","default":null},{"name":"kTrp","type":"long","doc":"BLD kTrp fS jty FBuKC"},{"name":"kdYkvr","type":"string","doc":"BLD name fS jty kdYkvr"},{"name":"jfqWhtm","type":"string","doc":"BLD name fS jty jfqWhtm"},{"name":"ORVhoRguhhZ","type":["nu [...]
\ No newline at end of file
diff --git a/gobblin-core-base/src/test/resources/converter/recursive_schema_2_not_converted.avsc b/gobblin-core-base/src/test/resources/converter/recursive_schema_2_not_converted.avsc
index 81519b3..c3130f9 100644
--- a/gobblin-core-base/src/test/resources/converter/recursive_schema_2_not_converted.avsc
+++ b/gobblin-core-base/src/test/resources/converter/recursive_schema_2_not_converted.avsc
@@ -1 +1 @@
-{"type":"record","name":"XQLFognzBcWX","namespace":"IoV.uIwHpaVy.dUV","fields":[{"name":"eFQjDj","type":["null",{"type":"record","name":"CWlSbaEOPjQ","fields":[{"name":"TzuYZajb","type":"int","doc":"BLD isOtza xj fS jty qGGt PpaPkENfDJ jty HTvfxa"},{"name":"zMMEITWsv","type":["null","string"],"doc":"Pnu fS jty qGGt PpaPkENfDJ jty HTvfxa","default":null},{"name":"kTrp","type":"long","doc":"BLD kTrp fS jty FBuKC"},{"name":"kdYkvr","type":"string","doc":"BLD name fS jty kdYkvr"},{"name":"jf [...]
\ No newline at end of file
+{"type":"record","name":"XQLFognzBcWX","namespace":"IoV.uIwHpaVy.dUV","fields":[{"name":"eFQjDj","type":["null",{"type":"record","name":"CWlSbaEOPjQ","fields":[{"name":"TzuYZajb","type":"int","doc":"BLD isOtza xj fS jty qGGt PpaPkENfDJ jty HTvfxa"},{"name":"zMMEITWsv","type":["null","string"],"doc":"Pnu fS jty qGGt PpaPkENfDJ jty HTvfxa","default":null},{"name":"kTrp","type":"long","doc":"BLD kTrp fS jty FBuKC"},{"name":"kdYkvr","type":"string","doc":"BLD name fS jty kdYkvr"},{"name":"jf [...]
\ No newline at end of file
diff --git a/gradle/scripts/dependencyDefinitions.gradle b/gradle/scripts/dependencyDefinitions.gradle
index b8ac319..7723a71 100644
--- a/gradle/scripts/dependencyDefinitions.gradle
+++ b/gradle/scripts/dependencyDefinitions.gradle
@@ -25,7 +25,7 @@ ext.externalDependency = [
"antlrRuntime": "org.antlr:antlr-runtime:3.5.2",
"avro": "org.apache.avro:avro:" + avroVersion,
"avroMapredH2": "org.apache.avro:avro-mapred:" + avroVersion,
- "avroCompatHelper": "com.linkedin.avroutil1:helper-all:0.2.71",
+ "avroCompatHelper": "com.linkedin.avroutil1:helper-all:0.2.74",
"avroCompiler": "org.apache.avro:avro-compiler:" + avroVersion,
"awsCore": "com.amazonaws:aws-java-sdk-core:" + awsVersion,
"awsAsg": "com.amazonaws:aws-java-sdk-autoscaling:" + awsVersion,