You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by mc...@apache.org on 2018/08/01 18:06:57 UTC
nifi git commit: NIFI-5475: Upgraded Hive 3 bundle to Apache Hive
3.1.0
Repository: nifi
Updated Branches:
refs/heads/master e62aa0252 -> b4894c557
NIFI-5475: Upgraded Hive 3 bundle to Apache Hive 3.1.0
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/b4894c55
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/b4894c55
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/b4894c55
Branch: refs/heads/master
Commit: b4894c5572fd81713de57d866e13bd72130c5157
Parents: e62aa02
Author: Matthew Burgess <ma...@apache.org>
Authored: Tue Jul 31 12:51:46 2018 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Wed Aug 1 13:42:31 2018 -0400
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java | 14 ++++++++++----
.../org/apache/nifi/processors/orc/PutORCTest.java | 8 ++++----
nifi-nar-bundles/nifi-hive-bundle/pom.xml | 5 ++---
3 files changed, 16 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/b4894c55/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java
index 7231421..2e6d2ca 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java
@@ -24,8 +24,8 @@ import org.apache.avro.util.Utf8;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
-import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.io.DateWritableV2;
+import org.apache.hadoop.hive.serde2.io.TimestampWritableV2;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector;
@@ -113,10 +113,16 @@ public class NiFiOrcUtils {
return new BytesWritable(((ByteBuffer) o).array());
}
if (o instanceof Timestamp) {
- return new TimestampWritable((Timestamp) o);
+ Timestamp t = (Timestamp) o;
+ org.apache.hadoop.hive.common.type.Timestamp timestamp = new org.apache.hadoop.hive.common.type.Timestamp();
+ timestamp.setTimeInMillis(t.getTime(), t.getNanos());
+ return new TimestampWritableV2(timestamp);
}
if (o instanceof Date) {
- return new DateWritable((Date) o);
+ Date d = (Date) o;
+ org.apache.hadoop.hive.common.type.Date date = new org.apache.hadoop.hive.common.type.Date();
+ date.setTimeInMillis(d.getTime());
+ return new DateWritableV2(date);
}
if (o instanceof Object[]) {
Object[] objArray = (Object[]) o;
http://git-wip-us.apache.org/repos/asf/nifi/blob/b4894c55/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/processors/orc/PutORCTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/processors/orc/PutORCTest.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/processors/orc/PutORCTest.java
index e1af5a1..552544f 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/processors/orc/PutORCTest.java
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/processors/orc/PutORCTest.java
@@ -25,9 +25,9 @@ import org.apache.hadoop.hive.ql.io.orc.OrcFile;
import org.apache.hadoop.hive.ql.io.orc.OrcStruct;
import org.apache.hadoop.hive.ql.io.orc.Reader;
import org.apache.hadoop.hive.ql.io.orc.RecordReader;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.io.DateWritableV2;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritableV2;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
@@ -246,8 +246,8 @@ public class PutORCTest {
verifyORCUsers(orcFile, 10, (x, currUser) -> {
assertEquals((int) currUser, ((IntWritable) x.get(0)).get());
assertEquals(timeMillis, ((IntWritable) x.get(1)).get());
- assertEquals(timestampMillis, ((TimestampWritable) x.get(2)).getTimestamp());
- assertEquals(dt, ((DateWritable) x.get(3)).get());
+ assertEquals(timestampMillis, ((TimestampWritableV2) x.get(2)).getTimestamp().toSqlTimestamp());
+ assertEquals(dt.toLocalDate().toEpochDay(), ((DateWritableV2) x.get(3)).get().toEpochDay());
assertEquals(dec, ((DoubleWritable) x.get(4)).get(), Double.MIN_VALUE);
return null;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b4894c55/nifi-nar-bundles/nifi-hive-bundle/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/pom.xml b/nifi-nar-bundles/nifi-hive-bundle/pom.xml
index 2bb421b..c67a267 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-hive-bundle/pom.xml
@@ -38,9 +38,8 @@
<properties>
<hive.version>1.2.1</hive.version>
<hive.hadoop.version>2.6.2</hive.hadoop.version>
- <hive3.version>3.0.0</hive3.version>
- <hive3.hadoop.version>3.0.0</hive3.hadoop.version>
- <orc.version>1.4.3</orc.version>
+ <hive3.version>3.1.0</hive3.version>
+ <hive3.hadoop.version>3.1.0</hive3.hadoop.version>
</properties>
<build>