You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2014/12/16 03:13:10 UTC
svn commit: r1645821 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/exec/DDLTask.java
test/queries/clientpositive/show_create_table_serde.q
test/results/clientpositive/show_create_table_serde.q.out
Author: navis
Date: Tue Dec 16 02:13:10 2014
New Revision: 1645821
URL: http://svn.apache.org/r1645821
Log:
HIVE-7270 : SerDe Properties are not considered by show create table Command (Navis reviewed by Jason Dere)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
hive/trunk/ql/src/test/queries/clientpositive/show_create_table_serde.q
hive/trunk/ql/src/test/results/clientpositive/show_create_table_serde.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1645821&r1=1645820&r2=1645821&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Tue Dec 16 02:13:10 2014
@@ -1848,6 +1848,14 @@ public class DDLTask extends Task<DDLWor
return 0;
}
+ private static final String[] DELIMITER_PREFIXES = new String[] {
+ "FIELDS TERMINATED BY",
+ "COLLECTION ITEMS TERMINATED BY",
+ "MAP KEYS TERMINATED BY",
+ "LINES TERMINATED BY",
+ "NULL DEFINED AS"
+ };
+
/**
* Write a statement of how to create a table to a file.
*
@@ -1989,80 +1997,65 @@ public class DDLTask extends Task<DDLWor
}
// Row format (SerDe)
- String tbl_row_format = "";
+ StringBuilder tbl_row_format = new StringBuilder();
StorageDescriptor sd = tbl.getTTable().getSd();
SerDeInfo serdeInfo = sd.getSerdeInfo();
- tbl_row_format += "ROW FORMAT";
+ tbl_row_format.append("ROW FORMAT");
if (tbl.getStorageHandler() == null) {
- if (serdeInfo.getParametersSize() > 1) {
+ Map<String, String> serdeParams = serdeInfo.getParameters();
+ String[] delimiters = new String[] {
+ serdeParams.remove(serdeConstants.FIELD_DELIM),
+ serdeParams.remove(serdeConstants.COLLECTION_DELIM),
+ serdeParams.remove(serdeConstants.MAPKEY_DELIM),
+ serdeParams.remove(serdeConstants.LINE_DELIM),
+ serdeParams.remove(serdeConstants.SERIALIZATION_NULL_FORMAT)
+ };
+ serdeParams.remove(serdeConstants.SERIALIZATION_FORMAT);
+ if (containsNonNull(delimiters)) {
// There is a "serialization.format" property by default,
// even with a delimited row format.
// But our result will only cover the following four delimiters.
- tbl_row_format += " DELIMITED \n";
- Map<String, String> delims = serdeInfo.getParameters();
+ tbl_row_format.append(" DELIMITED \n");
+
// Warn:
// If the four delimiters all exist in a CREATE TABLE query,
// this following order needs to be strictly followed,
// or the query will fail with a ParseException.
- if (delims.containsKey(serdeConstants.FIELD_DELIM)) {
- tbl_row_format += " FIELDS TERMINATED BY '" +
- escapeHiveCommand(StringEscapeUtils.escapeJava(delims.get(
- serdeConstants.FIELD_DELIM))) + "' \n";
- }
- if (delims.containsKey(serdeConstants.COLLECTION_DELIM)) {
- tbl_row_format += " COLLECTION ITEMS TERMINATED BY '" +
- escapeHiveCommand(StringEscapeUtils.escapeJava(delims.get(
- serdeConstants.COLLECTION_DELIM))) + "' \n";
- }
- if (delims.containsKey(serdeConstants.MAPKEY_DELIM)) {
- tbl_row_format += " MAP KEYS TERMINATED BY '" +
- escapeHiveCommand(StringEscapeUtils.escapeJava(delims.get(
- serdeConstants.MAPKEY_DELIM))) + "' \n";
- }
- if (delims.containsKey(serdeConstants.LINE_DELIM)) {
- tbl_row_format += " LINES TERMINATED BY '" +
- escapeHiveCommand(StringEscapeUtils.escapeJava(delims.get(
- serdeConstants.LINE_DELIM))) + "' \n";
- }
- if (delims.containsKey(serdeConstants.SERIALIZATION_NULL_FORMAT)) {
- tbl_row_format += " NULL DEFINED AS '" +
- escapeHiveCommand(StringEscapeUtils.escapeJava(delims.get(
- serdeConstants.SERIALIZATION_NULL_FORMAT))) + "' \n";
+ for (int i = 0; i < DELIMITER_PREFIXES.length; i++) {
+ if (delimiters[i] != null) {
+ tbl_row_format.append(" ").append(DELIMITER_PREFIXES[i]).append(" '");
+ tbl_row_format.append(escapeHiveCommand(StringEscapeUtils.escapeJava(delimiters[i])));
+ tbl_row_format.append("' \n");
+ }
}
+ } else {
+ tbl_row_format.append(" SERDE \n '" +
+ escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n");
}
- else {
- tbl_row_format += " SERDE \n '" +
- escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n";
+ if (!serdeParams.isEmpty()) {
+ appendSerdeParams(tbl_row_format, serdeParams).append(" \n");
}
- tbl_row_format += "STORED AS INPUTFORMAT \n '" +
- escapeHiveCommand(sd.getInputFormat()) + "' \n";
- tbl_row_format += "OUTPUTFORMAT \n '" +
- escapeHiveCommand(sd.getOutputFormat()) + "'";
- }
- else {
+ tbl_row_format.append("STORED AS INPUTFORMAT \n '" +
+ escapeHiveCommand(sd.getInputFormat()) + "' \n");
+ tbl_row_format.append("OUTPUTFORMAT \n '" +
+ escapeHiveCommand(sd.getOutputFormat()) + "'");
+ } else {
duplicateProps.add(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE);
- tbl_row_format += " SERDE \n '" +
- escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n";
- tbl_row_format += "STORED BY \n '" + escapeHiveCommand(tbl.getParameters().get(
- org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE)) + "' \n";
+ tbl_row_format.append(" SERDE \n '" +
+ escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n");
+ tbl_row_format.append("STORED BY \n '" + escapeHiveCommand(tbl.getParameters().get(
+ org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE)) + "' \n");
// SerDe Properties
if (serdeInfo.getParametersSize() > 0) {
- tbl_row_format += "WITH SERDEPROPERTIES ( \n";
- List<String> serdeCols = new ArrayList<String>();
- for (Map.Entry<String, String> entry : serdeInfo.getParameters().entrySet()) {
- serdeCols.add(" '" + entry.getKey() + "'='"
- + escapeHiveCommand(StringEscapeUtils.escapeJava(entry.getValue())) + "'");
- }
- tbl_row_format += StringUtils.join(serdeCols, ", \n");
- tbl_row_format += ")";
+ appendSerdeParams(tbl_row_format, serdeInfo.getParameters());
}
}
String tbl_location = " '" + escapeHiveCommand(sd.getLocation()) + "'";
// Table properties
String tbl_properties = "";
- Map<String, String> properties = new TreeMap<String, String>(tbl.getParameters());
- if (properties.size() > 0) {
+ if (!tbl.getParameters().isEmpty()) {
+ Map<String, String> properties = new TreeMap<String, String>(tbl.getParameters());
List<String> realProps = new ArrayList<String>();
for (String key : properties.keySet()) {
if (properties.get(key) != null && !duplicateProps.contains(key)) {
@@ -2104,6 +2097,27 @@ public class DDLTask extends Task<DDLWor
return 0;
}
+ private boolean containsNonNull(String[] values) {
+ for (String value : values) {
+ if (value != null) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private StringBuilder appendSerdeParams(StringBuilder builder, Map<String, String> serdeParam) {
+ serdeParam = new TreeMap<String, String>(serdeParam);
+ builder.append("WITH SERDEPROPERTIES ( \n");
+ List<String> serdeCols = new ArrayList<String>();
+ for (Entry<String, String> entry : serdeParam.entrySet()) {
+ serdeCols.add(" '" + entry.getKey() + "'='"
+ + escapeHiveCommand(StringEscapeUtils.escapeJava(entry.getValue())) + "'");
+ }
+ builder.append(StringUtils.join(serdeCols, ", \n")).append(')');
+ return builder;
+ }
+
/**
* Write a list of indexes to a file.
*
Modified: hive/trunk/ql/src/test/queries/clientpositive/show_create_table_serde.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/show_create_table_serde.q?rev=1645821&r1=1645820&r2=1645821&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/show_create_table_serde.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/show_create_table_serde.q Tue Dec 16 02:13:10 2014
@@ -1,5 +1,10 @@
-- Test SHOW CREATE TABLE on a table with serde.
+CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint);
+ALTER TABLE tmp_showcrt1 SET SERDEPROPERTIES ('custom.property.key1'='custom.property.value1', 'custom.property.key2'='custom.property.value2');
+SHOW CREATE TABLE tmp_showcrt1;
+DROP TABLE tmp_showcrt1;
+
-- without a storage handler
CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint)
COMMENT 'temporary table'
@@ -7,6 +12,15 @@ ROW FORMAT SERDE 'org.apache.hadoop.hive
STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat'
OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat';
SHOW CREATE TABLE tmp_showcrt1;
+DROP TABLE tmp_showcrt1;
+
+-- without a storage handler / with custom serde params
+CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint)
+ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'
+WITH SERDEPROPERTIES ('custom.property.key1'='custom.property.value1', 'custom.property.key2'='custom.property.value2')
+STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat';
+SHOW CREATE TABLE tmp_showcrt1;
DROP TABLE tmp_showcrt1;
-- with a storage handler and serde properties
Modified: hive/trunk/ql/src/test/results/clientpositive/show_create_table_serde.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/show_create_table_serde.q.out?rev=1645821&r1=1645820&r2=1645821&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/show_create_table_serde.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/show_create_table_serde.q.out Tue Dec 16 02:13:10 2014
@@ -1,6 +1,61 @@
PREHOOK: query: -- Test SHOW CREATE TABLE on a table with serde.
--- without a storage handler
+CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tmp_showcrt1
+POSTHOOK: query: -- Test SHOW CREATE TABLE on a table with serde.
+
+CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tmp_showcrt1
+PREHOOK: query: ALTER TABLE tmp_showcrt1 SET SERDEPROPERTIES ('custom.property.key1'='custom.property.value1', 'custom.property.key2'='custom.property.value2')
+PREHOOK: type: ALTERTABLE_SERDEPROPERTIES
+PREHOOK: Input: default@tmp_showcrt1
+PREHOOK: Output: default@tmp_showcrt1
+POSTHOOK: query: ALTER TABLE tmp_showcrt1 SET SERDEPROPERTIES ('custom.property.key1'='custom.property.value1', 'custom.property.key2'='custom.property.value2')
+POSTHOOK: type: ALTERTABLE_SERDEPROPERTIES
+POSTHOOK: Input: default@tmp_showcrt1
+POSTHOOK: Output: default@tmp_showcrt1
+PREHOOK: query: SHOW CREATE TABLE tmp_showcrt1
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: default@tmp_showcrt1
+POSTHOOK: query: SHOW CREATE TABLE tmp_showcrt1
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: default@tmp_showcrt1
+CREATE TABLE `tmp_showcrt1`(
+ `key` int,
+ `value` string,
+ `newvalue` bigint)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+WITH SERDEPROPERTIES (
+ 'custom.property.key1'='custom.property.value1',
+ 'custom.property.key2'='custom.property.value2')
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+#### A masked pattern was here ####
+TBLPROPERTIES (
+ 'COLUMN_STATS_ACCURATE'='false',
+#### A masked pattern was here ####
+ 'numFiles'='0',
+ 'numRows'='-1',
+ 'rawDataSize'='-1',
+ 'totalSize'='0',
+#### A masked pattern was here ####
+PREHOOK: query: DROP TABLE tmp_showcrt1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@tmp_showcrt1
+PREHOOK: Output: default@tmp_showcrt1
+POSTHOOK: query: DROP TABLE tmp_showcrt1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@tmp_showcrt1
+POSTHOOK: Output: default@tmp_showcrt1
+PREHOOK: query: -- without a storage handler
CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint)
COMMENT 'temporary table'
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'
@@ -9,9 +64,7 @@ OUTPUTFORMAT 'org.apache.hadoop.hive.ql.
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@tmp_showcrt1
-POSTHOOK: query: -- Test SHOW CREATE TABLE on a table with serde.
-
--- without a storage handler
+POSTHOOK: query: -- without a storage handler
CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint)
COMMENT 'temporary table'
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'
@@ -49,6 +102,55 @@ POSTHOOK: query: DROP TABLE tmp_showcrt1
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@tmp_showcrt1
POSTHOOK: Output: default@tmp_showcrt1
+PREHOOK: query: -- without a storage handler / with custom serde params
+CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint)
+ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'
+WITH SERDEPROPERTIES ('custom.property.key1'='custom.property.value1', 'custom.property.key2'='custom.property.value2')
+STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tmp_showcrt1
+POSTHOOK: query: -- without a storage handler / with custom serde params
+CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint)
+ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'
+WITH SERDEPROPERTIES ('custom.property.key1'='custom.property.value1', 'custom.property.key2'='custom.property.value2')
+STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tmp_showcrt1
+PREHOOK: query: SHOW CREATE TABLE tmp_showcrt1
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: default@tmp_showcrt1
+POSTHOOK: query: SHOW CREATE TABLE tmp_showcrt1
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: default@tmp_showcrt1
+CREATE TABLE `tmp_showcrt1`(
+ `key` int,
+ `value` string,
+ `newvalue` bigint)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'
+WITH SERDEPROPERTIES (
+ 'custom.property.key1'='custom.property.value1',
+ 'custom.property.key2'='custom.property.value2')
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.RCFileInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat'
+LOCATION
+#### A masked pattern was here ####
+TBLPROPERTIES (
+#### A masked pattern was here ####
+PREHOOK: query: DROP TABLE tmp_showcrt1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@tmp_showcrt1
+PREHOOK: Output: default@tmp_showcrt1
+POSTHOOK: query: DROP TABLE tmp_showcrt1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@tmp_showcrt1
+POSTHOOK: Output: default@tmp_showcrt1
PREHOOK: query: -- with a storage handler and serde properties
CREATE EXTERNAL TABLE tmp_showcrt1 (key string, value boolean)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'
@@ -79,8 +181,8 @@ ROW FORMAT SERDE
STORED BY
'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler'
WITH SERDEPROPERTIES (
- 'serialization.format'='$',
- 'field.delim'=',')
+ 'field.delim'=',',
+ 'serialization.format'='$')
LOCATION
#### A masked pattern was here ####
TBLPROPERTIES (