You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by hu...@apache.org on 2019/10/10 21:45:50 UTC
[incubator-gobblin] branch master updated: [GOBBLIN-896] Clone
schema and field props in AvroSchemaFieldRemover
This is an automated email from the ASF dual-hosted git repository.
hutran pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 928e018 [GOBBLIN-896] Clone schema and field props in AvroSchemaFieldRemover
928e018 is described below
commit 928e0180c471fc4b7a6caee041b001b5b34e1cc6
Author: zhchen <zh...@linkedin.com>
AuthorDate: Thu Oct 10 14:45:42 2019 -0700
[GOBBLIN-896] Clone schema and field props in AvroSchemaFieldRemover
Closes #2750 from zxcware/props
---
.../apache/gobblin/converter/filter/AvroSchemaFieldRemover.java | 7 +++++++
.../src/test/resources/converter/recursive_schema_1.avsc | 2 +-
.../src/test/resources/converter/recursive_schema_1_converted.avsc | 2 +-
.../src/test/resources/converter/recursive_schema_2_converted.avsc | 2 +-
.../test/resources/converter/recursive_schema_2_not_converted.avsc | 2 +-
5 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/gobblin-core-base/src/main/java/org/apache/gobblin/converter/filter/AvroSchemaFieldRemover.java b/gobblin-core-base/src/main/java/org/apache/gobblin/converter/filter/AvroSchemaFieldRemover.java
index d681619..22f4d80 100644
--- a/gobblin-core-base/src/main/java/org/apache/gobblin/converter/filter/AvroSchemaFieldRemover.java
+++ b/gobblin-core-base/src/main/java/org/apache/gobblin/converter/filter/AvroSchemaFieldRemover.java
@@ -22,6 +22,7 @@ import java.util.Map;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Field;
+import org.codehaus.jackson.JsonNode;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
@@ -105,6 +106,9 @@ public class AvroSchemaFieldRemover {
private Schema removeFieldsFromRecords(Schema schema, Map<String, Schema> schemaMap) {
Schema newRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError());
+ for (Map.Entry<String, JsonNode> stringJsonNodeEntry : schema.getJsonProps().entrySet()) {
+ newRecord.addProp(stringJsonNodeEntry.getKey(), stringJsonNodeEntry.getValue());
+ }
// Put an incomplete schema into schemaMap to avoid re-processing a recursive field.
// The fields in the incomplete schema will be populated once the current schema is completely processed.
@@ -121,6 +125,9 @@ public class AvroSchemaFieldRemover {
newField = new Field(field.name(), DO_NOTHING_INSTANCE.removeFields(field.schema(), schemaMap), field.doc(),
field.defaultValue());
}
+ for (Map.Entry<String, JsonNode> stringJsonNodeEntry : field.getJsonProps().entrySet()) {
+ newField.addProp(stringJsonNodeEntry.getKey(), stringJsonNodeEntry.getValue());
+ }
newFields.add(newField);
}
}
diff --git a/gobblin-core-base/src/test/resources/converter/recursive_schema_1.avsc b/gobblin-core-base/src/test/resources/converter/recursive_schema_1.avsc
index 1cece39..4b41e2d 100644
--- a/gobblin-core-base/src/test/resources/converter/recursive_schema_1.avsc
+++ b/gobblin-core-base/src/test/resources/converter/recursive_schema_1.avsc
@@ -1 +1 @@
-{"type":"record","name":"VyPswKoukcXEZshQrXnE","namespace":"PKA.POshikUo.flXRgM.aBxSQzgOe","fields":[{"name":"jRqjDF","type":{"type":"record","name":"ZDlOMWcUTCk","namespace":"PKA.POshikUo.flXRgM","fields":[{"name":"GJiZXGQc","type":"int","doc":"Vdj NXTFcrls GsWTlJ Lw WS oWm ZbYt OxeGXFqHeG oWm gQpsyC NXTFcrls GsWTlJ ISn yqw tVlMiJQo XECvKHj JUyZ BGwP PMEQNW yqw qfzNChDuzDx GwBGpC jM BGwP Ox r xOrwhzll lIVyeB"},{"name":"NZUOnQgci","type":["null","string"],"doc":"Vdj NXTFcrls gIn WS oWm Z [...]
+{"type":"record","name":"VyPswKoukcXEZshQrXnE","namespace":"PKA.POshikUo.flXRgM.aBxSQzgOe","fields":[{"name":"jRqjDF","type":{"type":"record","name":"ZDlOMWcUTCk","namespace":"PKA.POshikUo.flXRgM","fields":[{"name":"GJiZXGQc","type":"int","doc":"Vdj NXTFcrls GsWTlJ Lw WS oWm ZbYt OxeGXFqHeG oWm gQpsyC NXTFcrls GsWTlJ ISn yqw tVlMiJQo XECvKHj JUyZ BGwP PMEQNW yqw qfzNChDuzDx GwBGpC jM BGwP Ox r xOrwhzll lIVyeB","meta":"field_meta"},{"name":"NZUOnQgci","type":["null","string"],"doc":"Vdj N [...]
diff --git a/gobblin-core-base/src/test/resources/converter/recursive_schema_1_converted.avsc b/gobblin-core-base/src/test/resources/converter/recursive_schema_1_converted.avsc
index ca6ac1e..66a8fae 100644
--- a/gobblin-core-base/src/test/resources/converter/recursive_schema_1_converted.avsc
+++ b/gobblin-core-base/src/test/resources/converter/recursive_schema_1_converted.avsc
@@ -1 +1 @@
-{"type":"record","name":"VyPswKoukcXEZshQrXnE","namespace":"PKA.POshikUo.flXRgM.aBxSQzgOe","fields":[{"name":"jRqjDF","type":{"type":"record","name":"ZDlOMWcUTCk","namespace":"PKA.POshikUo.flXRgM","fields":[{"name":"GJiZXGQc","type":"int","doc":"Vdj NXTFcrls GsWTlJ Lw WS oWm ZbYt OxeGXFqHeG oWm gQpsyC NXTFcrls GsWTlJ ISn yqw tVlMiJQo XECvKHj JUyZ BGwP PMEQNW yqw qfzNChDuzDx GwBGpC jM BGwP Ox r xOrwhzll lIVyeB"},{"name":"NZUOnQgci","type":["null","string"],"doc":"Vdj NXTFcrls gIn WS oWm Z [...]
+{"type":"record","name":"VyPswKoukcXEZshQrXnE","namespace":"PKA.POshikUo.flXRgM.aBxSQzgOe","fields":[{"name":"jRqjDF","type":{"type":"record","name":"ZDlOMWcUTCk","namespace":"PKA.POshikUo.flXRgM","fields":[{"name":"GJiZXGQc","type":"int","doc":"Vdj NXTFcrls GsWTlJ Lw WS oWm ZbYt OxeGXFqHeG oWm gQpsyC NXTFcrls GsWTlJ ISn yqw tVlMiJQo XECvKHj JUyZ BGwP PMEQNW yqw qfzNChDuzDx GwBGpC jM BGwP Ox r xOrwhzll lIVyeB","meta":"field_meta"},{"name":"NZUOnQgci","type":["null","string"],"doc":"Vdj N [...]
diff --git a/gobblin-core-base/src/test/resources/converter/recursive_schema_2_converted.avsc b/gobblin-core-base/src/test/resources/converter/recursive_schema_2_converted.avsc
index 1e63079..a733019 100644
--- a/gobblin-core-base/src/test/resources/converter/recursive_schema_2_converted.avsc
+++ b/gobblin-core-base/src/test/resources/converter/recursive_schema_2_converted.avsc
@@ -1 +1 @@
-{"type":"record","name":"XQLFognzBcWX","namespace":"IoV.uIwHpaVy.dUV","fields":[{"name":"eFQjDj","type":["null",{"type":"record","name":"CWlSbaEOPjQ","fields":[{"name":"zMMEITWsv","type":["null","string"],"doc":"Pnu fS jty qGGt PpaPkENfDJ jty HTvfxa","default":null},{"name":"kTrp","type":"long","doc":"BLD kTrp fS jty FBuKC"},{"name":"kdYkvr","type":"string","doc":"BLD name fS jty kdYkvr"},{"name":"jfqWhtm","type":"string","doc":"BLD name fS jty jfqWhtm"},{"name":"ORVhoRguhhZ","type":["nu [...]
+{"type":"record","name":"XQLFognzBcWX","namespace":"IoV.uIwHpaVy.dUV","fields":[{"name":"eFQjDj","type":["null",{"type":"record","name":"CWlSbaEOPjQ","fields":[{"name":"zMMEITWsv","type":["null","string"],"doc":"Pnu fS jty qGGt PpaPkENfDJ jty HTvfxa","default":null},{"name":"kTrp","type":"long","doc":"BLD kTrp fS jty FBuKC"},{"name":"kdYkvr","type":"string","doc":"BLD name fS jty kdYkvr"},{"name":"jfqWhtm","type":"string","doc":"BLD name fS jty jfqWhtm"},{"name":"ORVhoRguhhZ","type":["nu [...]
\ No newline at end of file
diff --git a/gobblin-core-base/src/test/resources/converter/recursive_schema_2_not_converted.avsc b/gobblin-core-base/src/test/resources/converter/recursive_schema_2_not_converted.avsc
index 428bec3..c3130f9 100644
--- a/gobblin-core-base/src/test/resources/converter/recursive_schema_2_not_converted.avsc
+++ b/gobblin-core-base/src/test/resources/converter/recursive_schema_2_not_converted.avsc
@@ -1 +1 @@
-{"type":"record","name":"XQLFognzBcWX","namespace":"IoV.uIwHpaVy.dUV","fields":[{"name":"eFQjDj","type":["null",{"type":"record","name":"CWlSbaEOPjQ","fields":[{"name":"TzuYZajb","type":"int","doc":"BLD isOtza xj fS jty qGGt PpaPkENfDJ jty HTvfxa"},{"name":"zMMEITWsv","type":["null","string"],"doc":"Pnu fS jty qGGt PpaPkENfDJ jty HTvfxa","default":null},{"name":"kTrp","type":"long","doc":"BLD kTrp fS jty FBuKC"},{"name":"kdYkvr","type":"string","doc":"BLD name fS jty kdYkvr"},{"name":"jf [...]
\ No newline at end of file
+{"type":"record","name":"XQLFognzBcWX","namespace":"IoV.uIwHpaVy.dUV","fields":[{"name":"eFQjDj","type":["null",{"type":"record","name":"CWlSbaEOPjQ","fields":[{"name":"TzuYZajb","type":"int","doc":"BLD isOtza xj fS jty qGGt PpaPkENfDJ jty HTvfxa"},{"name":"zMMEITWsv","type":["null","string"],"doc":"Pnu fS jty qGGt PpaPkENfDJ jty HTvfxa","default":null},{"name":"kTrp","type":"long","doc":"BLD kTrp fS jty FBuKC"},{"name":"kdYkvr","type":"string","doc":"BLD name fS jty kdYkvr"},{"name":"jf [...]
\ No newline at end of file