You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by pv...@apache.org on 2018/04/19 07:50:06 UTC
nifi git commit: NIFI-4647: Fix support for strings in unions for
ConvertAvroToORC
Repository: nifi
Updated Branches:
refs/heads/master aa196bc01 -> 090e74872
NIFI-4647: Fix support for strings in unions for ConvertAvroToORC
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #2644.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/090e7487
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/090e7487
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/090e7487
Branch: refs/heads/master
Commit: 090e748726b319a96ec941f51299d1436f5f5136
Parents: aa196bc
Author: Matthew Burgess <ma...@apache.org>
Authored: Wed Apr 18 14:49:16 2018 -0400
Committer: Pierre Villard <pi...@gmail.com>
Committed: Thu Apr 19 09:49:32 2018 +0200
----------------------------------------------------------------------
.../hadoop/hive/ql/io/orc/NiFiOrcUtils.java | 8 +++++++-
.../apache/nifi/util/orc/TestNiFiOrcUtils.java | 19 +++++++++++++++++++
2 files changed, 26 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/090e7487/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java
index c9624b6..687073e 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java
@@ -67,9 +67,15 @@ public class NiFiOrcUtils {
if (o != null) {
if (typeInfo instanceof UnionTypeInfo) {
OrcUnion union = new OrcUnion();
+ // Avro uses Utf8 and GenericData.EnumSymbol objects instead of Strings. This is handled in other places in the method, but here
+ // we need to determine the union types from the objects, so choose String.class if the object is one of those Avro classes
+ Class clazzToCompareTo = o.getClass();
+ if (o instanceof org.apache.avro.util.Utf8 || o instanceof GenericData.EnumSymbol) {
+ clazzToCompareTo = String.class;
+ }
// Need to find which of the union types correspond to the primitive object
TypeInfo objectTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(
- ObjectInspectorFactory.getReflectionObjectInspector(o.getClass(), ObjectInspectorFactory.ObjectInspectorOptions.JAVA));
+ ObjectInspectorFactory.getReflectionObjectInspector(clazzToCompareTo, ObjectInspectorFactory.ObjectInspectorOptions.JAVA));
List<TypeInfo> unionTypeInfos = ((UnionTypeInfo) typeInfo).getAllUnionObjectTypeInfos();
int index = 0;
http://git-wip-us.apache.org/repos/asf/nifi/blob/090e7487/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/util/orc/TestNiFiOrcUtils.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/util/orc/TestNiFiOrcUtils.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/util/orc/TestNiFiOrcUtils.java
index 342aed9..47ee3a5 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/util/orc/TestNiFiOrcUtils.java
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/util/orc/TestNiFiOrcUtils.java
@@ -20,7 +20,9 @@ package org.apache.nifi.util.orc;
import org.apache.avro.Schema;
import org.apache.avro.SchemaBuilder;
import org.apache.avro.generic.GenericData;
+import org.apache.avro.util.Utf8;
import org.apache.hadoop.hive.ql.io.orc.NiFiOrcUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.UnionObject;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
@@ -267,6 +269,23 @@ public class TestNiFiOrcUtils {
+ " STORED AS ORC", ddl);
}
+ @Test
+ public void test_convertToORCObject() {
+ Schema schema = SchemaBuilder.enumeration("myEnum").symbols("x","y","z");
+ List<Object> objects = Arrays.asList(new Utf8("Hello"), new GenericData.EnumSymbol(schema, "x"));
+ objects.forEach((avroObject) -> {
+ Object o = NiFiOrcUtils.convertToORCObject(TypeInfoUtils.getTypeInfoFromTypeString("uniontype<bigint,string>"), avroObject);
+ assertTrue(o instanceof UnionObject);
+ UnionObject uo = (UnionObject) o;
+ assertTrue(uo.getObject() instanceof Text);
+ });
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void test_convertToORCObjectBadUnion() {
+ NiFiOrcUtils.convertToORCObject(TypeInfoUtils.getTypeInfoFromTypeString("uniontype<bigint,long>"), "Hello");
+ }
+
//////////////////
// Helper methods