You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ct...@apache.org on 2015/12/04 20:39:15 UTC
[2/2] hive git commit: HIVE-12506:SHOW CREATE TABLE command creates a
table that does not work for RCFile format (Chaoyu Tang,
reviewed by Jimmy Xiang)
HIVE-12506:SHOW CREATE TABLE command creates a table that does not work for RCFile format (Chaoyu Tang, reviewed by Jimmy Xiang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/09e7b19b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/09e7b19b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/09e7b19b
Branch: refs/heads/branch-1
Commit: 09e7b19bf0fc629291d1da5aea9f0c63baa4d952
Parents: 44eab8e
Author: ctang <ct...@gmail.com>
Authored: Fri Dec 4 11:04:14 2015 -0500
Committer: ctang <ct...@gmail.com>
Committed: Fri Dec 4 14:37:12 2015 -0500
----------------------------------------------------------------------
.../hadoop/hive/metastore/MetaStoreUtils.java | 5 +-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 51 ++++----------------
.../results/clientpositive/nullformat.q.out | 6 ++-
.../results/clientpositive/nullformatCTAS.q.out | 6 ++-
.../show_create_table_delimited.q.out | 13 +++--
5 files changed, 28 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/09e7b19b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 425549b..f5153b3 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -90,6 +90,7 @@ public class MetaStoreUtils {
public static final String DEFAULT_DATABASE_NAME = "default";
public static final String DEFAULT_DATABASE_COMMENT = "Default Hive database";
+ public static final String DEFAULT_SERIALIZATION_FORMAT = "1";
public static final String DATABASE_WAREHOUSE_SUFFIX = ".db";
@@ -108,8 +109,8 @@ public class MetaStoreUtils {
SerDeInfo serdeInfo = sd.getSerdeInfo();
serdeInfo.setSerializationLib(LazySimpleSerDe.class.getName());
serdeInfo.setParameters(new HashMap<String, String>());
- serdeInfo.getParameters().put(
- org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1");
+ serdeInfo.getParameters().put(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT,
+ DEFAULT_SERIALIZATION_FORMAT);
List<FieldSchema> fields = new ArrayList<FieldSchema>();
sd.setCols(fields);
http://git-wip-us.apache.org/repos/asf/hive/blob/09e7b19b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 78c7872..c3693de 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -2065,37 +2065,15 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
StringBuilder tbl_row_format = new StringBuilder();
StorageDescriptor sd = tbl.getTTable().getSd();
SerDeInfo serdeInfo = sd.getSerdeInfo();
- tbl_row_format.append("ROW FORMAT");
+ Map<String, String> serdeParams = serdeInfo.getParameters();
+ tbl_row_format.append("ROW FORMAT SERDE \n");
+ tbl_row_format.append(" '" + escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n");
if (tbl.getStorageHandler() == null) {
- Map<String, String> serdeParams = serdeInfo.getParameters();
- String[] delimiters = new String[] {
- serdeParams.remove(serdeConstants.FIELD_DELIM),
- serdeParams.remove(serdeConstants.COLLECTION_DELIM),
- serdeParams.remove(serdeConstants.MAPKEY_DELIM),
- serdeParams.remove(serdeConstants.LINE_DELIM),
- serdeParams.remove(serdeConstants.SERIALIZATION_NULL_FORMAT)
- };
- serdeParams.remove(serdeConstants.SERIALIZATION_FORMAT);
- if (containsNonNull(delimiters)) {
- // There is a "serialization.format" property by default,
- // even with a delimited row format.
- // But our result will only cover the following four delimiters.
- tbl_row_format.append(" DELIMITED \n");
-
- // Warn:
- // If the four delimiters all exist in a CREATE TABLE query,
- // this following order needs to be strictly followed,
- // or the query will fail with a ParseException.
- for (int i = 0; i < DELIMITER_PREFIXES.length; i++) {
- if (delimiters[i] != null) {
- tbl_row_format.append(" ").append(DELIMITER_PREFIXES[i]).append(" '");
- tbl_row_format.append(escapeHiveCommand(StringEscapeUtils.escapeJava(delimiters[i])));
- tbl_row_format.append("' \n");
- }
- }
- } else {
- tbl_row_format.append(" SERDE \n '" +
- escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n");
+ // If serialization.format property has the default value, it will not to be included in
+ // SERDE properties
+ if (MetaStoreUtils.DEFAULT_SERIALIZATION_FORMAT.equals(serdeParams.get(
+ serdeConstants.SERIALIZATION_FORMAT))){
+ serdeParams.remove(serdeConstants.SERIALIZATION_FORMAT);
}
if (!serdeParams.isEmpty()) {
appendSerdeParams(tbl_row_format, serdeParams).append(" \n");
@@ -2106,12 +2084,10 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
escapeHiveCommand(sd.getOutputFormat()) + "'");
} else {
duplicateProps.add(META_TABLE_STORAGE);
- tbl_row_format.append(" SERDE \n '" +
- escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n");
tbl_row_format.append("STORED BY \n '" + escapeHiveCommand(tbl.getParameters().get(
META_TABLE_STORAGE)) + "' \n");
// SerDe Properties
- if (serdeInfo.getParametersSize() > 0) {
+ if (!serdeParams.isEmpty()) {
appendSerdeParams(tbl_row_format, serdeInfo.getParameters());
}
}
@@ -2162,15 +2138,6 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
return 0;
}
- private boolean containsNonNull(String[] values) {
- for (String value : values) {
- if (value != null) {
- return true;
- }
- }
- return false;
- }
-
private StringBuilder appendSerdeParams(StringBuilder builder, Map<String, String> serdeParam) {
serdeParam = new TreeMap<String, String>(serdeParam);
builder.append("WITH SERDEPROPERTIES ( \n");
http://git-wip-us.apache.org/repos/asf/hive/blob/09e7b19b/ql/src/test/results/clientpositive/nullformat.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/nullformat.q.out b/ql/src/test/results/clientpositive/nullformat.q.out
index 6cfc2b8..af91470 100644
--- a/ql/src/test/results/clientpositive/nullformat.q.out
+++ b/ql/src/test/results/clientpositive/nullformat.q.out
@@ -82,8 +82,10 @@ POSTHOOK: Input: default@null_tab1
CREATE TABLE `null_tab1`(
`a` string,
`b` string)
-ROW FORMAT DELIMITED
- NULL DEFINED AS 'fooNull'
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+WITH SERDEPROPERTIES (
+ 'serialization.null.format'='fooNull')
STORED AS INPUTFORMAT
'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT
http://git-wip-us.apache.org/repos/asf/hive/blob/09e7b19b/ql/src/test/results/clientpositive/nullformatCTAS.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/nullformatCTAS.q.out b/ql/src/test/results/clientpositive/nullformatCTAS.q.out
index 7adca68..d6b22b1 100644
--- a/ql/src/test/results/clientpositive/nullformatCTAS.q.out
+++ b/ql/src/test/results/clientpositive/nullformatCTAS.q.out
@@ -163,8 +163,10 @@ POSTHOOK: Input: default@null_tab3
CREATE TABLE `null_tab3`(
`a` string,
`b` string)
-ROW FORMAT DELIMITED
- NULL DEFINED AS 'fooNull'
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+WITH SERDEPROPERTIES (
+ 'serialization.null.format'='fooNull')
STORED AS INPUTFORMAT
'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT
http://git-wip-us.apache.org/repos/asf/hive/blob/09e7b19b/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/show_create_table_delimited.q.out b/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
index 79f29de..e14f850 100644
--- a/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
+++ b/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
@@ -28,11 +28,14 @@ CREATE TABLE `tmp_showcrt1`(
`key` int,
`value` string,
`newvalue` bigint)
-ROW FORMAT DELIMITED
- FIELDS TERMINATED BY ','
- COLLECTION ITEMS TERMINATED BY '|'
- MAP KEYS TERMINATED BY '%'
- LINES TERMINATED BY '\n'
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+WITH SERDEPROPERTIES (
+ 'colelction.delim'='|',
+ 'field.delim'=',',
+ 'line.delim'='\n',
+ 'mapkey.delim'='%',
+ 'serialization.format'=',')
STORED AS INPUTFORMAT
'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT