You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by ab...@apache.org on 2017/08/28 20:09:19 UTC
incubator-gobblin git commit: [GOBBLIN-228] Add config property to
ignore fields
Repository: incubator-gobblin
Updated Branches:
refs/heads/master d769b2144 -> 396fc40de
[GOBBLIN-228] Add config property to ignore fields
Closes #2082 from jack-moseley/master
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/396fc40d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/396fc40d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/396fc40d
Branch: refs/heads/master
Commit: 396fc40dee6d4f7dac6029acdba3043305740ee8
Parents: d769b21
Author: Jack Moseley <jm...@linkedin.com>
Authored: Mon Aug 28 13:09:20 2017 -0700
Committer: Abhishek Tiwari <ab...@gmail.com>
Committed: Mon Aug 28 13:09:20 2017 -0700
----------------------------------------------------------------------
.../avro/JsonRecordAvroSchemaToAvroConverter.java | 13 ++++++++++++-
.../avro/JsonRecordAvroSchemaToAvroConverterTest.java | 4 +++-
.../src/test/resources/converter/jsonToAvroSchema.avsc | 4 ++++
3 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/396fc40d/gobblin-core/src/main/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverter.java
----------------------------------------------------------------------
diff --git a/gobblin-core/src/main/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverter.java b/gobblin-core/src/main/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverter.java
index c3edd25..6f5e7b3 100644
--- a/gobblin-core/src/main/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverter.java
+++ b/gobblin-core/src/main/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverter.java
@@ -17,7 +17,6 @@
package org.apache.gobblin.converter.avro;
-import com.google.common.base.Preconditions;
import java.util.List;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
@@ -29,6 +28,8 @@ import org.apache.gobblin.converter.SchemaConversionException;
import org.apache.gobblin.converter.SingleRecordIterable;
import org.apache.gobblin.converter.ToAvroConverterBase;
import com.google.gson.JsonObject;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
/**
@@ -37,14 +38,19 @@ import com.google.gson.JsonObject;
*/
public class JsonRecordAvroSchemaToAvroConverter<SI> extends ToAvroConverterBase<SI, JsonObject> {
+ private static final Splitter SPLITTER_ON_COMMA = Splitter.on(',').trimResults().omitEmptyStrings();
+
public static final String AVRO_SCHEMA_KEY = "gobblin.converter.avroSchema";
+ public static final String IGNORE_FIELDS = "gobblin.converter.ignoreFields";
private Schema schema;
+ private List<String> ignoreFields;
public ToAvroConverterBase<SI, JsonObject> init(WorkUnitState workUnit) {
super.init(workUnit);
Preconditions.checkArgument(workUnit.contains(AVRO_SCHEMA_KEY));
this.schema = new Schema.Parser().parse(workUnit.getProp(AVRO_SCHEMA_KEY));
+ this.ignoreFields = SPLITTER_ON_COMMA.splitToList(workUnit.getProp(IGNORE_FIELDS, ""));
return this;
}
@@ -70,6 +76,11 @@ public class JsonRecordAvroSchemaToAvroConverter<SI> extends ToAvroConverterBase
GenericRecord avroRecord = new GenericData.Record(outputSchema);
JsonElementConversionWithAvroSchemaFactory.JsonElementConverter converter;
for (Schema.Field field : outputSchema.getFields()) {
+
+ if (this.ignoreFields.contains(field.name())) {
+ continue;
+ }
+
if (inputRecord.get(field.name()) == null) {
throw new DataConversionException("Field missing from record: " + field.name());
}
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/396fc40d/gobblin-core/src/test/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverterTest.java
----------------------------------------------------------------------
diff --git a/gobblin-core/src/test/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverterTest.java b/gobblin-core/src/test/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverterTest.java
index 7da37b7..9971d83 100644
--- a/gobblin-core/src/test/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverterTest.java
+++ b/gobblin-core/src/test/java/org/apache/gobblin/converter/avro/JsonRecordAvroSchemaToAvroConverterTest.java
@@ -53,12 +53,13 @@ public class JsonRecordAvroSchemaToAvroConverterTest {
this.state = new WorkUnitState(
source.createWorkUnit(source.createExtract(TableType.SNAPSHOT_ONLY, "test_table", "test_namespace")));
this.state.setProp(JsonRecordAvroSchemaToAvroConverter.AVRO_SCHEMA_KEY, avroSchemaString);
+ this.state.setProp(JsonRecordAvroSchemaToAvroConverter.IGNORE_FIELDS, "fieldToIgnore");
}
@Test
public void testConverter()
throws Exception {
- JsonRecordAvroSchemaToAvroConverter<String> converter = new JsonRecordAvroSchemaToAvroConverter();
+ JsonRecordAvroSchemaToAvroConverter<String> converter = new JsonRecordAvroSchemaToAvroConverter<>();
converter.init(this.state);
@@ -66,6 +67,7 @@ public class JsonRecordAvroSchemaToAvroConverterTest {
GenericRecord record = converter.convertRecord(avroSchema, this.jsonRecord, this.state).iterator().next();
+ Assert.assertEquals(record.get("fieldToIgnore"), null);
Assert.assertEquals(record.get("nullableField"), null);
Assert.assertEquals(record.get("longField"), 1234L);
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/396fc40d/gobblin-core/src/test/resources/converter/jsonToAvroSchema.avsc
----------------------------------------------------------------------
diff --git a/gobblin-core/src/test/resources/converter/jsonToAvroSchema.avsc b/gobblin-core/src/test/resources/converter/jsonToAvroSchema.avsc
index 22fec73..effc91c 100644
--- a/gobblin-core/src/test/resources/converter/jsonToAvroSchema.avsc
+++ b/gobblin-core/src/test/resources/converter/jsonToAvroSchema.avsc
@@ -4,6 +4,10 @@
"namespace": "org.apache.gobblin.test",
"fields": [
{
+ "name": "fieldToIgnore",
+ "type": "string"
+ },
+ {
"name": "nullableField",
"type": ["string", "null"]
},