You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ai...@apache.org on 2016/12/22 18:42:30 UTC
hive git commit: HIVE-15113: SHOW CREATE TABLE on skewed table
returns statement without skew definition (Aihua Xu, reviewed by Yongzhi Chen)
Repository: hive
Updated Branches:
refs/heads/master 4ba713ccd -> 71f250db3
HIVE-15113: SHOW CREATE TABLE on skewed table returns statement without skew definition (Aihua Xu, reviewed by Yongzhi Chen)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/71f250db
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/71f250db
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/71f250db
Branch: refs/heads/master
Commit: 71f250db341f788f69bea6a6ea2d5ca3f0070240
Parents: 4ba713c
Author: Aihua Xu <ai...@apache.org>
Authored: Mon Dec 19 16:09:53 2016 -0500
Committer: Aihua Xu <ai...@apache.org>
Committed: Thu Dec 22 13:41:24 2016 -0500
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 19 ++++
.../clientpositive/show_create_table_db_table.q | 12 ++-
.../show_create_table_db_table.q.out | 105 ++++++++++++++++---
3 files changed, 118 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/71f250db/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 2bc90a5..a1fb874 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -2109,6 +2109,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
final String TBL_COMMENT = "tbl_comment";
final String LIST_PARTITIONS = "partitions";
final String SORT_BUCKET = "sort_bucket";
+ final String SKEWED_INFO = "tbl_skewedinfo";
final String ROW_FORMAT = "row_format";
final String TBL_LOCATION = "tbl_location";
final String TBL_PROPERTIES = "tbl_properties";
@@ -2133,6 +2134,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
createTab_str.append("<" + TBL_COMMENT + ">\n");
createTab_str.append("<" + LIST_PARTITIONS + ">\n");
createTab_str.append("<" + SORT_BUCKET + ">\n");
+ createTab_str.append("<" + SKEWED_INFO + ">\n");
createTab_str.append("<" + ROW_FORMAT + ">\n");
if (needsLocation) {
createTab_str.append("LOCATION\n");
@@ -2225,6 +2227,22 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
tbl_sort_bucket += "INTO " + tbl.getNumBuckets() + " BUCKETS";
}
+ // Skewed Info
+ StringBuilder tbl_skewedinfo = new StringBuilder();
+ SkewedInfo skewedInfo = tbl.getSkewedInfo();
+ if (skewedInfo != null && !skewedInfo.getSkewedColNames().isEmpty()) {
+ tbl_skewedinfo.append("SKEWED BY (" + StringUtils.join(skewedInfo.getSkewedColNames(), ",") + ")\n");
+ tbl_skewedinfo.append(" ON (");
+ List<String> colValueList = new ArrayList<String>();
+ for (List<String> colValues : skewedInfo.getSkewedColValues()) {
+ colValueList.add("('" + StringUtils.join(colValues, "','") + "')");
+ }
+ tbl_skewedinfo.append(StringUtils.join(colValueList, ",") + ")");
+ if (tbl.isStoredAsSubDirectories()) {
+ tbl_skewedinfo.append("\n STORED AS DIRECTORIES");
+ }
+ }
+
// Row format (SerDe)
StringBuilder tbl_row_format = new StringBuilder();
StorageDescriptor sd = tbl.getTTable().getSd();
@@ -2269,6 +2287,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
createTab_stmt.add(TBL_COMMENT, tbl_comment);
createTab_stmt.add(LIST_PARTITIONS, tbl_partitions);
createTab_stmt.add(SORT_BUCKET, tbl_sort_bucket);
+ createTab_stmt.add(SKEWED_INFO, tbl_skewedinfo);
createTab_stmt.add(ROW_FORMAT, tbl_row_format);
// Table location should not be printed with hbase backed tables
if (needsLocation) {
http://git-wip-us.apache.org/repos/asf/hive/blob/71f250db/ql/src/test/queries/clientpositive/show_create_table_db_table.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/show_create_table_db_table.q b/ql/src/test/queries/clientpositive/show_create_table_db_table.q
index 1937acb..35ac5bf 100644
--- a/ql/src/test/queries/clientpositive/show_create_table_db_table.q
+++ b/ql/src/test/queries/clientpositive/show_create_table_db_table.q
@@ -2,9 +2,15 @@
CREATE DATABASE tmp_feng comment 'for show create table test';
SHOW DATABASES;
-CREATE TABLE tmp_feng.tmp_showcrt (key string, value int);
+CREATE TABLE tmp_feng.tmp_showcrt1(key string, value int);
+CREATE TABLE tmp_feng.tmp_showcrt2(key string, value int) skewed by (key) on ('1','2');
+CREATE TABLE tmp_feng.tmp_showcrt3(key string, value int) skewed by (key) on ('1','2') stored as directories;
USE default;
-SHOW CREATE TABLE tmp_feng.tmp_showcrt;
-DROP TABLE tmp_feng.tmp_showcrt;
+SHOW CREATE TABLE tmp_feng.tmp_showcrt1;
+SHOW CREATE TABLE tmp_feng.tmp_showcrt2;
+SHOW CREATE TABLE tmp_feng.tmp_showcrt3;
+DROP TABLE tmp_feng.tmp_showcrt1;
+DROP TABLE tmp_feng.tmp_showcrt2;
+DROP TABLE tmp_feng.tmp_showcrt3;
DROP DATABASE tmp_feng;
http://git-wip-us.apache.org/repos/asf/hive/blob/71f250db/ql/src/test/results/clientpositive/show_create_table_db_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/show_create_table_db_table.q.out b/ql/src/test/results/clientpositive/show_create_table_db_table.q.out
index 495f4b5..41ccb56 100644
--- a/ql/src/test/results/clientpositive/show_create_table_db_table.q.out
+++ b/ql/src/test/results/clientpositive/show_create_table_db_table.q.out
@@ -14,29 +14,88 @@ POSTHOOK: query: SHOW DATABASES
POSTHOOK: type: SHOWDATABASES
default
tmp_feng
-PREHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt (key string, value int)
+PREHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt1(key string, value int)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:tmp_feng
-PREHOOK: Output: tmp_feng@tmp_showcrt
-POSTHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt (key string, value int)
+PREHOOK: Output: tmp_feng@tmp_showcrt1
+POSTHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt1(key string, value int)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:tmp_feng
-POSTHOOK: Output: tmp_feng@tmp_showcrt
+POSTHOOK: Output: tmp_feng@tmp_showcrt1
+PREHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt2(key string, value int) skewed by (key) on ('1','2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:tmp_feng
+PREHOOK: Output: tmp_feng@tmp_showcrt2
+POSTHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt2(key string, value int) skewed by (key) on ('1','2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:tmp_feng
+POSTHOOK: Output: tmp_feng@tmp_showcrt2
+PREHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt3(key string, value int) skewed by (key) on ('1','2') stored as directories
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:tmp_feng
+PREHOOK: Output: tmp_feng@tmp_showcrt3
+POSTHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt3(key string, value int) skewed by (key) on ('1','2') stored as directories
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:tmp_feng
+POSTHOOK: Output: tmp_feng@tmp_showcrt3
PREHOOK: query: USE default
PREHOOK: type: SWITCHDATABASE
PREHOOK: Input: database:default
POSTHOOK: query: USE default
POSTHOOK: type: SWITCHDATABASE
POSTHOOK: Input: database:default
-PREHOOK: query: SHOW CREATE TABLE tmp_feng.tmp_showcrt
+PREHOOK: query: SHOW CREATE TABLE tmp_feng.tmp_showcrt1
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: tmp_feng@tmp_showcrt1
+POSTHOOK: query: SHOW CREATE TABLE tmp_feng.tmp_showcrt1
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: tmp_feng@tmp_showcrt1
+CREATE TABLE `tmp_feng.tmp_showcrt1`(
+ `key` string,
+ `value` int)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+#### A masked pattern was here ####
+TBLPROPERTIES (
+#### A masked pattern was here ####
+PREHOOK: query: SHOW CREATE TABLE tmp_feng.tmp_showcrt2
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: tmp_feng@tmp_showcrt2
+POSTHOOK: query: SHOW CREATE TABLE tmp_feng.tmp_showcrt2
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: tmp_feng@tmp_showcrt2
+CREATE TABLE `tmp_feng.tmp_showcrt2`(
+ `key` string,
+ `value` int)
+SKEWED BY (key)
+ ON (('1'),('2'))
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+#### A masked pattern was here ####
+TBLPROPERTIES (
+#### A masked pattern was here ####
+PREHOOK: query: SHOW CREATE TABLE tmp_feng.tmp_showcrt3
PREHOOK: type: SHOW_CREATETABLE
-PREHOOK: Input: tmp_feng@tmp_showcrt
-POSTHOOK: query: SHOW CREATE TABLE tmp_feng.tmp_showcrt
+PREHOOK: Input: tmp_feng@tmp_showcrt3
+POSTHOOK: query: SHOW CREATE TABLE tmp_feng.tmp_showcrt3
POSTHOOK: type: SHOW_CREATETABLE
-POSTHOOK: Input: tmp_feng@tmp_showcrt
-CREATE TABLE `tmp_feng.tmp_showcrt`(
+POSTHOOK: Input: tmp_feng@tmp_showcrt3
+CREATE TABLE `tmp_feng.tmp_showcrt3`(
`key` string,
`value` int)
+SKEWED BY (key)
+ ON (('1'),('2'))
+ STORED AS DIRECTORIES
ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
STORED AS INPUTFORMAT
@@ -47,14 +106,30 @@ LOCATION
#### A masked pattern was here ####
TBLPROPERTIES (
#### A masked pattern was here ####
-PREHOOK: query: DROP TABLE tmp_feng.tmp_showcrt
+PREHOOK: query: DROP TABLE tmp_feng.tmp_showcrt1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: tmp_feng@tmp_showcrt1
+PREHOOK: Output: tmp_feng@tmp_showcrt1
+POSTHOOK: query: DROP TABLE tmp_feng.tmp_showcrt1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: tmp_feng@tmp_showcrt1
+POSTHOOK: Output: tmp_feng@tmp_showcrt1
+PREHOOK: query: DROP TABLE tmp_feng.tmp_showcrt2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: tmp_feng@tmp_showcrt2
+PREHOOK: Output: tmp_feng@tmp_showcrt2
+POSTHOOK: query: DROP TABLE tmp_feng.tmp_showcrt2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: tmp_feng@tmp_showcrt2
+POSTHOOK: Output: tmp_feng@tmp_showcrt2
+PREHOOK: query: DROP TABLE tmp_feng.tmp_showcrt3
PREHOOK: type: DROPTABLE
-PREHOOK: Input: tmp_feng@tmp_showcrt
-PREHOOK: Output: tmp_feng@tmp_showcrt
-POSTHOOK: query: DROP TABLE tmp_feng.tmp_showcrt
+PREHOOK: Input: tmp_feng@tmp_showcrt3
+PREHOOK: Output: tmp_feng@tmp_showcrt3
+POSTHOOK: query: DROP TABLE tmp_feng.tmp_showcrt3
POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: tmp_feng@tmp_showcrt
-POSTHOOK: Output: tmp_feng@tmp_showcrt
+POSTHOOK: Input: tmp_feng@tmp_showcrt3
+POSTHOOK: Output: tmp_feng@tmp_showcrt3
PREHOOK: query: DROP DATABASE tmp_feng
PREHOOK: type: DROPDATABASE
PREHOOK: Input: database:tmp_feng