You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/11/11 07:08:01 UTC
svn commit: r1638032 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java
test/queries/clientpositive/create_like.q
test/results/clientpositive/create_like.q.out
Author: brock
Date: Tue Nov 11 06:08:01 2014
New Revision: 1638032
URL: http://svn.apache.org/r1638032
Log:
HIVE-8469 - Add parquet.compression as a Serde Property (Ferdinand Xu via Brock)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java
hive/trunk/ql/src/test/queries/clientpositive/create_like.q
hive/trunk/ql/src/test/results/clientpositive/create_like.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java?rev=1638032&r1=1638031&r2=1638032&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java Tue Nov 11 06:08:01 2014
@@ -63,6 +63,7 @@ import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
+import parquet.hadoop.ParquetOutputFormat;
import parquet.io.api.Binary;
/**
@@ -70,13 +71,17 @@ import parquet.io.api.Binary;
* A ParquetHiveSerDe for Hive (with the deprecated package mapred)
*
*/
-@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES})
+@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES,
+ ParquetOutputFormat.COMPRESSION})
public class ParquetHiveSerDe extends AbstractSerDe {
public static final Text MAP_KEY = new Text("key");
public static final Text MAP_VALUE = new Text("value");
public static final Text MAP = new Text("map");
public static final Text ARRAY = new Text("bag");
+ // default compression type is uncompressed
+ private static final String DEFAULTCOMPRESSION = "UNCOMPRESSED";
+
// Map precision to the number bytes needed for binary conversion.
public static final int PRECISION_TO_BYTE_COUNT[] = new int[38];
static {
@@ -99,6 +104,7 @@ public class ParquetHiveSerDe extends Ab
private LAST_OPERATION status;
private long serializedSize;
private long deserializedSize;
+ private String compressionType;
@Override
public final void initialize(final Configuration conf, final Properties tbl) throws SerDeException {
@@ -110,6 +116,9 @@ public class ParquetHiveSerDe extends Ab
final String columnNameProperty = tbl.getProperty(serdeConstants.LIST_COLUMNS);
final String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES);
+ // Get compression properties
+ compressionType = tbl.getProperty(ParquetOutputFormat.COMPRESSION, DEFAULTCOMPRESSION);
+
if (columnNameProperty.length() == 0) {
columnNames = new ArrayList<String>();
} else {
Modified: hive/trunk/ql/src/test/queries/clientpositive/create_like.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/create_like.q?rev=1638032&r1=1638031&r2=1638032&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/create_like.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/create_like.q Tue Nov 11 06:08:01 2014
@@ -64,3 +64,9 @@ DESCRIBE FORMATTED doctors;
CREATE TABLE doctors2 like doctors;
DESCRIBE FORMATTED doctors2;
+
+CREATE TABLE CompressedParquetTable(a INT, b STRING) STORED AS PARQUET TBLPROPERTIES("parquet.compression"="LZO");
+CREATE TABLE LikeCompressedParquetTable LIKE CompressedParquetTable;
+
+DESCRIBE FORMATTED LikeCompressedParquetTable;
+
Modified: hive/trunk/ql/src/test/results/clientpositive/create_like.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/create_like.q.out?rev=1638032&r1=1638031&r2=1638032&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/create_like.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/create_like.q.out Tue Nov 11 06:08:01 2014
@@ -405,3 +405,51 @@ Bucket Columns: []
Sort Columns: []
Storage Desc Params:
serialization.format 1
+PREHOOK: query: CREATE TABLE CompressedParquetTable(a INT, b STRING) STORED AS PARQUET TBLPROPERTIES("parquet.compression"="LZO")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@CompressedParquetTable
+POSTHOOK: query: CREATE TABLE CompressedParquetTable(a INT, b STRING) STORED AS PARQUET TBLPROPERTIES("parquet.compression"="LZO")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@CompressedParquetTable
+PREHOOK: query: CREATE TABLE LikeCompressedParquetTable LIKE CompressedParquetTable
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@LikeCompressedParquetTable
+POSTHOOK: query: CREATE TABLE LikeCompressedParquetTable LIKE CompressedParquetTable
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@LikeCompressedParquetTable
+PREHOOK: query: DESCRIBE FORMATTED LikeCompressedParquetTable
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@likecompressedparquettable
+POSTHOOK: query: DESCRIBE FORMATTED LikeCompressedParquetTable
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@likecompressedparquettable
+# col_name data_type comment
+
+a int
+b string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ parquet.compression LZO
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
+InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1