You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@nifi.apache.org by "hipotures (Jira)" <ji...@apache.org> on 2020/12/07 17:10:00 UTC
[jira] [Updated] (NIFI-8075) ConvertRecord Json to Avro with null
values
[ https://issues.apache.org/jira/browse/NIFI-8075?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
hipotures updated NIFI-8075:
----------------------------
Description:
ConvertRecord processor with settings
Record Reader: JsonTreeReader (with Infer Schema)
Record Writer: AvroRecordSetWriter (with Embed Avro Schema)
change non {color:#000000}schema field names{color} avro (e.g. with "-", "@") to fields with underscore, but with NULL values.
Example below.
Input JSON:
{noformat}
{
"id":"6e07e04d-d644-bf76-f30f-ce648da478e0",
"type":"agreement",
"attributes":{
"code":"16-00-04332-20-01",
"origin-code":"07-19-305",
"service-type":"07",
"service-name":"Leczenie stomatologiczne",
"amount":112500.0,
"updated-at":"2019-12-13T08:39:20",
"provider-code":"160004332",
"provider-name":"ISP STOMATOLOGICZNA JANUSZ KUBRAK",
"provider-place":"SZCZECIN",
"year":2020,
"branch":"16"
},
"links":{
"related":"https://api.nfz.gov.pl/app-umw-api/agreements/6e07e04d-d644-bf76-f30f-ce648da478e0?format=json"
}
}{noformat}
Output Avro data:
{code:java}
[
{
"id":"6e07e04d-d644-bf76-f30f-ce648da478e0",
"type":"agreement",
"attributes":{
"code":"16-00-04332-20-01",
"origin_code":null,
"service_type":null,
"service_name":null,
"amount":112500.0,
"updated_at":null,
"provider_code":null,
"provider_name":null,
"provider_place":null,
"year":2020,
"branch":"16"
},
"links":{
"related":"https://api.nfz.gov.pl/app-umw-api/agreements/6e07e04d-d644-bf76-f30f-ce648da478e0?format=json"
}
}
]{code}
Output Avro schema:
{noformat}
{
"type":"record",
"name":"nifiRecord",
"namespace":"org.apache.nifi",
"fields":[
{
"name":"id",
"type":[
"null",
"string"
]
},
{
"name":"type",
"type":[
"null",
"string"
]
},
{
"name":"attributes",
"type":[
"null",
{
"type":"record",
"name":"attributesType",
"fields":[
{
"name":"code",
"type":[
"null",
"string"
]
},
{
"name":"origin_code",
"type":[
"null",
"string"
],
"aliases":[
"origin-code"
]
},
{
"name":"service_type",
"type":[
"null",
"string"
],
"aliases":[
"service-type"
]
},
{
"name":"service_name",
"type":[
"null",
"string"
],
"aliases":[
"service-name"
]
},
{
"name":"amount",
"type":[
"null",
"double"
]
},
{
"name":"updated_at",
"type":[
"null",
"string"
],
"aliases":[
"updated-at"
]
},
{
"name":"provider_code",
"type":[
"null",
"string"
],
"aliases":[
"provider-code"
]
},
{
"name":"provider_name",
"type":[
"null",
"string"
],
"aliases":[
"provider-name"
]
},
{
"name":"provider_place",
"type":[
"null",
"string"
],
"aliases":[
"provider-place"
]
},
{
"name":"year",
"type":[
"null",
"long"
]
},
{
"name":"branch",
"type":[
"null",
"string"
]
}
]
}
]
},
{
"name":"links",
"type":[
"null",
{
"type":"record",
"name":"linksType",
"fields":[
{
"name":"related",
"type":[
"null",
"string"
]
}
]
}
]
}
]
}{noformat}
Workaround: for "static" JSONs - ReplaceTextWithMapping processor before ConvertRecord.
was:
ConvertRecord processor with settings
Record Reader: JsonTreeReader (with Infer Schema)
Record Writer: AvroRecordSetWriter (with Embed Avro Schema)
change non {color:#000000}schema field names{color} avro (e.g. with "-", "@") to fields with underscore, but with NULL values.
Example below.
Input JSON:
{noformat}
{
"id" : "6e07e04d-d644-bf76-f30f-ce648da478e0",
"type" : "agreement",
"attributes" : {
"code" : "16-00-04332-20-01",
"origin-code" : "07-19-305",
"service-type" : "07",
"service-name" : "Leczenie stomatologiczne",
"amount" : 112500.0,
"updated-at" : "2019-12-13T08:39:20",
"provider-code" : "160004332",
"provider-name" : "ISP STOMATOLOGICZNA JANUSZ KUBRAK",
"provider-place" : "SZCZECIN",
"year" : 2020,
"branch" : "16"
},
"links" : {
"related" : "https://api.nfz.gov.pl/app-umw-api/agreements/6e07e04d-d644-bf76-f30f-ce648da478e0?format=json"
}
}{noformat}
Output Avro data:
{code:java}
[ {
"id" : "6e07e04d-d644-bf76-f30f-ce648da478e0",
"type" : "agreement",
"attributes" : {
"code" : "16-00-04332-20-01",
"origin_code" : null,
"service_type" : null,
"service_name" : null,
"amount" : 112500.0,
"updated_at" : null,
"provider_code" : null,
"provider_name" : null,
"provider_place" : null,
"year" : 2020,
"branch" : "16"
},
"links" : {
"related" : "https://api.nfz.gov.pl/app-umw-api/agreements/6e07e04d-d644-bf76-f30f-ce648da478e0?format=json"
}
} ]{code}
Output Avro schema:
{noformat}
{
"type":"record",
"name":"nifiRecord",
"namespace":"org.apache.nifi",
"fields":[
{
"name":"id",
"type":[
"null",
"string"
]
},
{
"name":"type",
"type":[
"null",
"string"
]
},
{
"name":"attributes",
"type":[
"null",
{
"type":"record",
"name":"attributesType",
"fields":[
{
"name":"code",
"type":[
"null",
"string"
]
},
{
"name":"origin_code",
"type":[
"null",
"string"
],
"aliases":[
"origin-code"
]
},
{
"name":"service_type",
"type":[
"null",
"string"
],
"aliases":[
"service-type"
]
},
{
"name":"service_name",
"type":[
"null",
"string"
],
"aliases":[
"service-name"
]
},
{
"name":"amount",
"type":[
"null",
"double"
]
},
{
"name":"updated_at",
"type":[
"null",
"string"
],
"aliases":[
"updated-at"
]
},
{
"name":"provider_code",
"type":[
"null",
"string"
],
"aliases":[
"provider-code"
]
},
{
"name":"provider_name",
"type":[
"null",
"string"
],
"aliases":[
"provider-name"
]
},
{
"name":"provider_place",
"type":[
"null",
"string"
],
"aliases":[
"provider-place"
]
},
{
"name":"year",
"type":[
"null",
"long"
]
},
{
"name":"branch",
"type":[
"null",
"string"
]
}
]
}
]
},
{
"name":"links",
"type":[
"null",
{
"type":"record",
"name":"linksType",
"fields":[
{
"name":"related",
"type":[
"null",
"string"
]
}
]
}
]
}
]
}{noformat}
Workaround: for "static" JSONs - ReplaceTextWithMapping processor before ConvertRecord.
> ConvertRecord Json to Avro with null values
> -------------------------------------------
>
> Key: NIFI-8075
> URL: https://issues.apache.org/jira/browse/NIFI-8075
> Project: Apache NiFi
> Issue Type: Bug
> Components: Core Framework
> Affects Versions: 1.12.1
> Environment: 18.04.1-Ubuntu
> Reporter: hipotures
> Priority: Major
>
> ConvertRecord processor with settings
> Record Reader: JsonTreeReader (with Infer Schema)
> Record Writer: AvroRecordSetWriter (with Embed Avro Schema)
> change non {color:#000000}schema field names{color} avro (e.g. with "-", "@") to fields with underscore, but with NULL values.
> Example below.
> Input JSON:
> {noformat}
> {
> "id":"6e07e04d-d644-bf76-f30f-ce648da478e0",
> "type":"agreement",
> "attributes":{
> "code":"16-00-04332-20-01",
> "origin-code":"07-19-305",
> "service-type":"07",
> "service-name":"Leczenie stomatologiczne",
> "amount":112500.0,
> "updated-at":"2019-12-13T08:39:20",
> "provider-code":"160004332",
> "provider-name":"ISP STOMATOLOGICZNA JANUSZ KUBRAK",
> "provider-place":"SZCZECIN",
> "year":2020,
> "branch":"16"
> },
> "links":{
> "related":"https://api.nfz.gov.pl/app-umw-api/agreements/6e07e04d-d644-bf76-f30f-ce648da478e0?format=json"
> }
> }{noformat}
> Output Avro data:
> {code:java}
> [
> {
> "id":"6e07e04d-d644-bf76-f30f-ce648da478e0",
> "type":"agreement",
> "attributes":{
> "code":"16-00-04332-20-01",
> "origin_code":null,
> "service_type":null,
> "service_name":null,
> "amount":112500.0,
> "updated_at":null,
> "provider_code":null,
> "provider_name":null,
> "provider_place":null,
> "year":2020,
> "branch":"16"
> },
> "links":{
> "related":"https://api.nfz.gov.pl/app-umw-api/agreements/6e07e04d-d644-bf76-f30f-ce648da478e0?format=json"
> }
> }
> ]{code}
> Output Avro schema:
> {noformat}
> {
> "type":"record",
> "name":"nifiRecord",
> "namespace":"org.apache.nifi",
> "fields":[
> {
> "name":"id",
> "type":[
> "null",
> "string"
> ]
> },
> {
> "name":"type",
> "type":[
> "null",
> "string"
> ]
> },
> {
> "name":"attributes",
> "type":[
> "null",
> {
> "type":"record",
> "name":"attributesType",
> "fields":[
> {
> "name":"code",
> "type":[
> "null",
> "string"
> ]
> },
> {
> "name":"origin_code",
> "type":[
> "null",
> "string"
> ],
> "aliases":[
> "origin-code"
> ]
> },
> {
> "name":"service_type",
> "type":[
> "null",
> "string"
> ],
> "aliases":[
> "service-type"
> ]
> },
> {
> "name":"service_name",
> "type":[
> "null",
> "string"
> ],
> "aliases":[
> "service-name"
> ]
> },
> {
> "name":"amount",
> "type":[
> "null",
> "double"
> ]
> },
> {
> "name":"updated_at",
> "type":[
> "null",
> "string"
> ],
> "aliases":[
> "updated-at"
> ]
> },
> {
> "name":"provider_code",
> "type":[
> "null",
> "string"
> ],
> "aliases":[
> "provider-code"
> ]
> },
> {
> "name":"provider_name",
> "type":[
> "null",
> "string"
> ],
> "aliases":[
> "provider-name"
> ]
> },
> {
> "name":"provider_place",
> "type":[
> "null",
> "string"
> ],
> "aliases":[
> "provider-place"
> ]
> },
> {
> "name":"year",
> "type":[
> "null",
> "long"
> ]
> },
> {
> "name":"branch",
> "type":[
> "null",
> "string"
> ]
> }
> ]
> }
> ]
> },
> {
> "name":"links",
> "type":[
> "null",
> {
> "type":"record",
> "name":"linksType",
> "fields":[
> {
> "name":"related",
> "type":[
> "null",
> "string"
> ]
> }
> ]
> }
> ]
> }
> ]
> }{noformat}
>
> Workaround: for "static" JSONs - ReplaceTextWithMapping processor before ConvertRecord.
>
>
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)