You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@nifi.apache.org by GitBox <gi...@apache.org> on 2019/09/11 15:19:12 UTC
[GitHub] [nifi] mattyb149 commented on a change in pull request #3684:
NIFI-6295: Refactored NiFiRecordSerDe to handle nested complex types
mattyb149 commented on a change in pull request #3684: NIFI-6295: Refactored NiFiRecordSerDe to handle nested complex types
URL: https://github.com/apache/nifi/pull/3684#discussion_r323303903
##########
File path: nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/hive/streaming/NiFiRecordSerDe.java
##########
@@ -187,152 +182,153 @@ private Object extractCurrentField(Record record, RecordField field, TypeInfo fi
}
switch (primitiveCategory) {
case BYTE:
- Integer bIntValue = record.getAsInt(fieldName);
- val = bIntValue == null ? null : bIntValue.byteValue();
+ Integer bIntValue = DataTypeUtils.toInteger(fieldValue, fieldName);
+ val = bIntValue.byteValue();
break;
case SHORT:
- Integer sIntValue = record.getAsInt(fieldName);
- val = sIntValue == null ? null : sIntValue.shortValue();
+ Integer sIntValue = DataTypeUtils.toInteger(fieldValue, fieldName);
+ val = sIntValue.shortValue();
break;
case INT:
- val = record.getAsInt(fieldName);
+ val = DataTypeUtils.toInteger(fieldValue, fieldName);
break;
case LONG:
- val = record.getAsLong(fieldName);
+ val = DataTypeUtils.toLong(fieldValue, fieldName);
break;
case BOOLEAN:
- val = record.getAsBoolean(fieldName);
+ val = DataTypeUtils.toBoolean(fieldValue, fieldName);
break;
case FLOAT:
- val = record.getAsFloat(fieldName);
+ val = DataTypeUtils.toFloat(fieldValue, fieldName);
break;
case DOUBLE:
- val = record.getAsDouble(fieldName);
+ val = DataTypeUtils.toDouble(fieldValue, fieldName);
break;
case STRING:
case VARCHAR:
case CHAR:
- val = record.getAsString(fieldName);
+ val = DataTypeUtils.toString(fieldValue, fieldName);
break;
case BINARY:
- Object[] array = record.getAsArray(fieldName);
- if (array == null) {
- return null;
+ final ArrayDataType arrayDataType;
+ if(fieldValue instanceof String) {
+ // Treat this as an array of bytes
+ arrayDataType = (ArrayDataType) RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.BYTE.getDataType());
+ } else {
+ arrayDataType = (ArrayDataType) fieldDataType;
}
+ Object[] array = DataTypeUtils.toArray(fieldValue, fieldName, arrayDataType.getElementType());
val = AvroTypeUtil.convertByteArray(array).array();
break;
case DATE:
- Date d = record.getAsDate(fieldName, field.getDataType().getFormat());
- if(d != null) {
- org.apache.hadoop.hive.common.type.Date hiveDate = new org.apache.hadoop.hive.common.type.Date();
- hiveDate.setTimeInMillis(d.getTime());
- val = hiveDate;
- } else {
- val = null;
- }
+ Date d = DataTypeUtils.toDate(fieldValue, () -> DataTypeUtils.getDateFormat(fieldDataType.getFormat()), fieldName);
+ org.apache.hadoop.hive.common.type.Date hiveDate = new org.apache.hadoop.hive.common.type.Date();
+ hiveDate.setTimeInMillis(d.getTime());
+ val = hiveDate;
break;
// ORC doesn't currently handle TIMESTAMPLOCALTZ
case TIMESTAMP:
- Timestamp ts = DataTypeUtils.toTimestamp(record.getValue(fieldName), () -> DataTypeUtils.getDateFormat(field.getDataType().getFormat()), fieldName);
- if(ts != null) {
- // Convert to Hive's Timestamp type
- org.apache.hadoop.hive.common.type.Timestamp hivetimestamp = new org.apache.hadoop.hive.common.type.Timestamp();
- hivetimestamp.setTimeInMillis(ts.getTime(), ts.getNanos());
- val = hivetimestamp;
- } else {
- val = null;
- }
+ Timestamp ts = DataTypeUtils.toTimestamp(fieldValue, () -> DataTypeUtils.getDateFormat(fieldDataType.getFormat()), fieldName);
+ // Convert to Hive's Timestamp type
+ org.apache.hadoop.hive.common.type.Timestamp hivetimestamp = new org.apache.hadoop.hive.common.type.Timestamp();
+ hivetimestamp.setTimeInMillis(ts.getTime(), ts.getNanos());
+ val = hivetimestamp;
break;
case DECIMAL:
- Double value = record.getAsDouble(fieldName);
- val = value == null ? null : HiveDecimal.create(value);
+ if(fieldValue instanceof BigDecimal){
+ val = HiveDecimal.create((BigDecimal) fieldValue);
+ } else if (fieldValue instanceof Double){
Review comment:
Hmm `.doubleValue()` takes care of it, maybe this was just to avoid the extra call? I think it's more confusing than helpful in this case, we can handle all Numbers with `.doubleValue()`, will remove, nice catch!
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services