You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by su...@apache.org on 2015/08/11 18:58:58 UTC
hive git commit: HIVE-11340 - Create ORC based table using like
clause doesn't copy compression property (Yongzhi Chen, reviewed by Chao Sun)
Repository: hive
Updated Branches:
refs/heads/master cfe9e484f -> 57ba795cb
HIVE-11340 - Create ORC based table using like clause doesn't copy compression property (Yongzhi Chen, reviewed by Chao Sun)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/57ba795c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/57ba795c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/57ba795c
Branch: refs/heads/master
Commit: 57ba795cbf98f275b7bae75669d8769aa35d9ee5
Parents: cfe9e48
Author: Yongzhi Chen <yo...@hotmail.com>
Authored: Tue Aug 11 09:58:30 2015 -0700
Committer: Chao Sun <su...@apache.org>
Committed: Tue Aug 11 09:58:30 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/io/orc/OrcConf.java | 2 +-
.../apache/hadoop/hive/ql/io/orc/OrcSerde.java | 6 +-
.../test/queries/clientpositive/create_like.q | 12 ++++
.../results/clientpositive/create_like.q.out | 66 ++++++++++++++++++++
4 files changed, 84 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/57ba795c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcConf.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcConf.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcConf.java
index 81b822f..132889c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcConf.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcConf.java
@@ -133,7 +133,7 @@ public enum OrcConf {
private String lookupValue(Properties tbl, Configuration conf) {
String result = null;
if (tbl != null) {
- result = conf.get(attribute);
+ result = tbl.getProperty(attribute);
}
if (result == null && conf != null) {
result = conf.get(attribute);
http://git-wip-us.apache.org/repos/asf/hive/blob/57ba795c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
index a381443..8beff4b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
@@ -22,6 +22,7 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Properties;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -42,7 +43,7 @@ import org.apache.hadoop.io.Writable;
* A serde class for ORC.
* It transparently passes the object to/from the ORC file reader/writer.
*/
-@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES})
+@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES, OrcSerde.COMPRESSION})
public class OrcSerde implements SerDe, VectorizedSerde {
private static final Log LOG = LogFactory.getLog(OrcSerde.class);
@@ -51,6 +52,7 @@ public class OrcSerde implements SerDe, VectorizedSerde {
private ObjectInspector inspector = null;
private VectorizedOrcSerde vos = null;
+ public static final String COMPRESSION = "orc.compress";
final class OrcSerdeRow implements Writable {
Object realRow;
@@ -82,6 +84,8 @@ public class OrcSerde implements SerDe, VectorizedSerde {
// NOTE: if "columns.types" is missing, all columns will be of String type
String columnTypeProperty = table.getProperty(serdeConstants.LIST_COLUMN_TYPES);
+ String compressType = OrcConf.COMPRESS.getString(table, conf);
+
// Parse the configuration parameters
ArrayList<String> columnNames = new ArrayList<String>();
if (columnNameProperty != null && columnNameProperty.length() > 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/57ba795c/ql/src/test/queries/clientpositive/create_like.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/create_like.q b/ql/src/test/queries/clientpositive/create_like.q
index 3b04702..bd39731 100644
--- a/ql/src/test/queries/clientpositive/create_like.q
+++ b/ql/src/test/queries/clientpositive/create_like.q
@@ -83,3 +83,15 @@ DESCRIBE FORMATTED table6;
drop table table5;
+create table orc_table (
+time string)
+stored as ORC tblproperties ("orc.compress"="SNAPPY");
+
+create table orc_table_using_like like orc_table;
+
+describe formatted orc_table_using_like;
+
+drop table orc_table_using_like;
+
+drop table orc_table;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/57ba795c/ql/src/test/results/clientpositive/create_like.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/create_like.q.out b/ql/src/test/results/clientpositive/create_like.q.out
index c93b134..a373178 100644
--- a/ql/src/test/results/clientpositive/create_like.q.out
+++ b/ql/src/test/results/clientpositive/create_like.q.out
@@ -579,3 +579,69 @@ POSTHOOK: query: drop table table5
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@table5
POSTHOOK: Output: default@table5
+PREHOOK: query: create table orc_table (
+time string)
+stored as ORC tblproperties ("orc.compress"="SNAPPY")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_table
+POSTHOOK: query: create table orc_table (
+time string)
+stored as ORC tblproperties ("orc.compress"="SNAPPY")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_table
+PREHOOK: query: create table orc_table_using_like like orc_table
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_table_using_like
+POSTHOOK: query: create table orc_table_using_like like orc_table
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_table_using_like
+PREHOOK: query: describe formatted orc_table_using_like
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@orc_table_using_like
+POSTHOOK: query: describe formatted orc_table_using_like
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@orc_table_using_like
+# col_name data_type comment
+
+time string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ orc.compress SNAPPY
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: drop table orc_table_using_like
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orc_table_using_like
+PREHOOK: Output: default@orc_table_using_like
+POSTHOOK: query: drop table orc_table_using_like
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@orc_table_using_like
+POSTHOOK: Output: default@orc_table_using_like
+PREHOOK: query: drop table orc_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orc_table
+PREHOOK: Output: default@orc_table
+POSTHOOK: query: drop table orc_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@orc_table
+POSTHOOK: Output: default@orc_table