You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ku...@apache.org on 2018/08/27 08:27:52 UTC
hive git commit: HIVE-20374: Write Hive version information to
Parquet footer (Laszlo Pinter via Marta Kuczora)
Repository: hive
Updated Branches:
refs/heads/master 1cb7e7734 -> f5e62eb03
HIVE-20374: Write Hive version information to Parquet footer (Laszlo Pinter via Marta Kuczora)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f5e62eb0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f5e62eb0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f5e62eb0
Branch: refs/heads/master
Commit: f5e62eb0387b94d135a4a67af2c21d3cc4db5398
Parents: 1cb7e77
Author: Laszlo Pinter <lp...@cloudera.com>
Authored: Mon Aug 27 10:27:04 2018 +0200
Committer: Marta Kuczora <ku...@cloudera.com>
Committed: Mon Aug 27 10:27:04 2018 +0200
----------------------------------------------------------------------
.../parquet/write/DataWritableWriteSupport.java | 6 ++++++
.../clientpositive/parquet_analyze.q.out | 4 ++--
.../parquet_vectorization_0.q.out | 20 ++++++++++----------
.../spark/parquet_vectorization_0.q.out | 20 ++++++++++----------
4 files changed, 28 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f5e62eb0/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriteSupport.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriteSupport.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriteSupport.java
index d13b898..8960944 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriteSupport.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriteSupport.java
@@ -18,6 +18,7 @@ import java.util.HashMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde2.io.ParquetHiveRecord;
+import org.apache.hive.common.util.HiveVersionInfo;
import org.apache.parquet.hadoop.api.WriteSupport;
import org.apache.parquet.io.api.RecordConsumer;
import org.apache.parquet.schema.MessageType;
@@ -58,4 +59,9 @@ public class DataWritableWriteSupport extends WriteSupport<ParquetHiveRecord> {
public void write(final ParquetHiveRecord record) {
writer.write(record);
}
+
+ @Override
+ public String getName() {
+ return HiveVersionInfo.getVersion();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f5e62eb0/ql/src/test/results/clientpositive/parquet_analyze.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_analyze.q.out b/ql/src/test/results/clientpositive/parquet_analyze.q.out
index e746621..16c836d 100644
--- a/ql/src/test/results/clientpositive/parquet_analyze.q.out
+++ b/ql/src/test/results/clientpositive/parquet_analyze.q.out
@@ -94,7 +94,7 @@ Table Parameters:
numFiles 1
numRows 100
rawDataSize 700
- totalSize 6692
+ totalSize 6730
#### A masked pattern was here ####
# Storage Information
@@ -142,7 +142,7 @@ Table Parameters:
numFiles 1
numRows 100
rawDataSize 5936
- totalSize 6692
+ totalSize 6730
#### A masked pattern was here ####
# Storage Information
http://git-wip-us.apache.org/repos/asf/hive/blob/f5e62eb0/ql/src/test/results/clientpositive/parquet_vectorization_0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_vectorization_0.q.out b/ql/src/test/results/clientpositive/parquet_vectorization_0.q.out
index 6af7892..4156c5d9 100644
--- a/ql/src/test/results/clientpositive/parquet_vectorization_0.q.out
+++ b/ql/src/test/results/clientpositive/parquet_vectorization_0.q.out
@@ -1744,7 +1744,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
@@ -1766,7 +1766,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
name: default.alltypesparquet
@@ -30565,7 +30565,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
@@ -30587,7 +30587,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
name: default.alltypesparquet
@@ -30678,7 +30678,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
@@ -30700,7 +30700,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
name: default.alltypesparquet
@@ -30795,7 +30795,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
@@ -30817,7 +30817,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
name: default.alltypesparquet
@@ -30897,7 +30897,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
@@ -30919,7 +30919,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
name: default.alltypesparquet
http://git-wip-us.apache.org/repos/asf/hive/blob/f5e62eb0/ql/src/test/results/clientpositive/spark/parquet_vectorization_0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/parquet_vectorization_0.q.out b/ql/src/test/results/clientpositive/spark/parquet_vectorization_0.q.out
index 268790a..288c38d 100644
--- a/ql/src/test/results/clientpositive/spark/parquet_vectorization_0.q.out
+++ b/ql/src/test/results/clientpositive/spark/parquet_vectorization_0.q.out
@@ -1835,7 +1835,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
@@ -1857,7 +1857,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
name: default.alltypesparquet
@@ -30661,7 +30661,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
@@ -30683,7 +30683,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
name: default.alltypesparquet
@@ -30777,7 +30777,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
@@ -30799,7 +30799,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
name: default.alltypesparquet
@@ -30897,7 +30897,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
@@ -30919,7 +30919,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
name: default.alltypesparquet
@@ -31004,7 +31004,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
@@ -31026,7 +31026,7 @@ STAGE PLANS:
serialization.ddl struct alltypesparquet { byte ctinyint, i16 csmallint, i32 cint, i64 cbigint, float cfloat, double cdouble, string cstring1, string cstring2, timestamp ctimestamp1, timestamp ctimestamp2, bool cboolean1, bool cboolean2}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
- totalSize 595103
+ totalSize 595141
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
name: default.alltypesparquet