You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mg...@apache.org on 2020/04/13 17:58:05 UTC
[hive] branch master updated: HIVE-23179 Show create table is not
showing SerDe Properties in unicode (Naresh P R, reviewed by Miklos Gergely)
This is an automated email from the ASF dual-hosted git repository.
mgergely pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 37ac058 HIVE-23179 Show create table is not showing SerDe Properties in unicode (Naresh P R, reviewed by Miklos Gergely)
37ac058 is described below
commit 37ac05861660e9863d51fb7cc34a0dcf13c5f1e5
Author: Naresh P R <pr...@gmail.com>
AuthorDate: Mon Apr 13 19:53:13 2020 +0200
HIVE-23179 Show create table is not showing SerDe Properties in unicode (Naresh P R, reviewed by Miklos Gergely)
---
.../apache/hive/common/util/HiveStringUtils.java | 14 ++
.../create/show/ShowCreateTableOperation.java | 10 +-
.../clientpositive/show_create_table_delimited.q | 15 ++
.../show_create_table_delimited.q.out | 156 +++++++++++++++++++++
4 files changed, 190 insertions(+), 5 deletions(-)
diff --git a/common/src/java/org/apache/hive/common/util/HiveStringUtils.java b/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
index 22948e3..6499ac1 100644
--- a/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
+++ b/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
@@ -43,6 +43,7 @@ import com.google.common.base.Splitter;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.text.translate.CharSequenceTranslator;
import org.apache.commons.lang3.text.translate.EntityArrays;
+import org.apache.commons.lang3.text.translate.JavaUnicodeEscaper;
import org.apache.commons.lang3.text.translate.LookupTranslator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
@@ -83,6 +84,9 @@ public class HiveStringUtils {
}).with(
new LookupTranslator(EntityArrays.JAVA_CTRL_CHARS_ESCAPE()));
+ private static final CharSequenceTranslator UNICODE_CONVERTER =
+ JavaUnicodeEscaper.outsideOf(32, 127);
+
static {
NumberFormat numberFormat = NumberFormat.getNumberInstance(Locale.ENGLISH);
decimalFormat = (DecimalFormat) numberFormat;
@@ -652,6 +656,16 @@ public class HiveStringUtils {
}
/**
+ * Escape java unicode characters.
+ *
+ * @param str Original string
+ * @return Escaped string
+ */
+ public static String escapeUnicode(String str) {
+ return UNICODE_CONVERTER.translate(str);
+ }
+
+ /**
* Unescape commas in the string using the default escape char
* @param str a string
* @return an unescaped string
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/show/ShowCreateTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/show/ShowCreateTableOperation.java
index 51d9f10..bf91344 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/show/ShowCreateTableOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/show/ShowCreateTableOperation.java
@@ -18,10 +18,6 @@
package org.apache.hadoop.hive.ql.ddl.table.create.show;
-import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
-import org.apache.hadoop.hive.ql.ddl.DDLUtils;
-import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperation;
-
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE;
import java.io.DataOutputStream;
@@ -47,6 +43,9 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.ql.ddl.DDLOperation;
+import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
+import org.apache.hadoop.hive.ql.ddl.DDLUtils;
+import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperation;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.util.DirectionUtils;
@@ -322,7 +321,8 @@ public class ShowCreateTableOperation extends DDLOperation<ShowCreateTableDesc>
SortedMap<String, String> sortedSerdeParams = new TreeMap<String, String>(serdeParams);
List<String> serdeCols = new ArrayList<String>();
for (Entry<String, String> entry : sortedSerdeParams.entrySet()) {
- serdeCols.add(" '" + entry.getKey() + "'='" + HiveStringUtils.escapeHiveCommand(entry.getValue()) + "'");
+ serdeCols.add(" '" + entry.getKey() + "'='" +
+ HiveStringUtils.escapeUnicode(HiveStringUtils.escapeHiveCommand(entry.getValue())) + "'");
}
builder
diff --git a/ql/src/test/queries/clientpositive/show_create_table_delimited.q b/ql/src/test/queries/clientpositive/show_create_table_delimited.q
index 7722964..4eef9d5 100644
--- a/ql/src/test/queries/clientpositive/show_create_table_delimited.q
+++ b/ql/src/test/queries/clientpositive/show_create_table_delimited.q
@@ -7,3 +7,18 @@ LOCATION 'file:${system:test.tmp.dir}/tmp_showcrt1';
SHOW CREATE TABLE tmp_showcrt1;
DROP TABLE tmp_showcrt1;
+create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\u0001' stored as textfile;
+show create table esc_special_delimiter;
+drop table esc_special_delimiter;
+
+create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '`' stored as textfile;
+show create table esc_special_delimiter;
+drop table esc_special_delimiter;
+
+create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\'' stored as textfile;
+show create table esc_special_delimiter;
+drop table esc_special_delimiter;
+
+create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';' stored as textfile;
+show create table esc_special_delimiter;
+drop table esc_special_delimiter;
diff --git a/ql/src/test/results/clientpositive/show_create_table_delimited.q.out b/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
index ad58bf6..fb88edb 100644
--- a/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
+++ b/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
@@ -49,3 +49,159 @@ POSTHOOK: query: DROP TABLE tmp_showcrt1
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@tmp_showcrt1
POSTHOOK: Output: default@tmp_showcrt1
+PREHOOK: query: create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\u0001' stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@esc_special_delimiter
+POSTHOOK: query: create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\u0001' stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@esc_special_delimiter
+PREHOOK: query: show create table esc_special_delimiter
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: default@esc_special_delimiter
+POSTHOOK: query: show create table esc_special_delimiter
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: default@esc_special_delimiter
+CREATE TABLE `esc_special_delimiter`(
+ `age` int,
+ `name` string)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+WITH SERDEPROPERTIES (
+ 'field.delim'='\u0001',
+ 'serialization.format'='\u0001')
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+#### A masked pattern was here ####
+TBLPROPERTIES (
+ 'bucketing_version'='2',
+#### A masked pattern was here ####
+PREHOOK: query: drop table esc_special_delimiter
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@esc_special_delimiter
+PREHOOK: Output: default@esc_special_delimiter
+POSTHOOK: query: drop table esc_special_delimiter
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@esc_special_delimiter
+POSTHOOK: Output: default@esc_special_delimiter
+PREHOOK: query: create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '`' stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@esc_special_delimiter
+POSTHOOK: query: create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '`' stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@esc_special_delimiter
+PREHOOK: query: show create table esc_special_delimiter
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: default@esc_special_delimiter
+POSTHOOK: query: show create table esc_special_delimiter
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: default@esc_special_delimiter
+CREATE TABLE `esc_special_delimiter`(
+ `age` int,
+ `name` string)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+WITH SERDEPROPERTIES (
+ 'field.delim'='`',
+ 'serialization.format'='`')
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+#### A masked pattern was here ####
+TBLPROPERTIES (
+ 'bucketing_version'='2',
+#### A masked pattern was here ####
+PREHOOK: query: drop table esc_special_delimiter
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@esc_special_delimiter
+PREHOOK: Output: default@esc_special_delimiter
+POSTHOOK: query: drop table esc_special_delimiter
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@esc_special_delimiter
+POSTHOOK: Output: default@esc_special_delimiter
+PREHOOK: query: create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\'' stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@esc_special_delimiter
+POSTHOOK: query: create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\'' stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@esc_special_delimiter
+PREHOOK: query: show create table esc_special_delimiter
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: default@esc_special_delimiter
+POSTHOOK: query: show create table esc_special_delimiter
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: default@esc_special_delimiter
+CREATE TABLE `esc_special_delimiter`(
+ `age` int,
+ `name` string)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+WITH SERDEPROPERTIES (
+ 'field.delim'='\'',
+ 'serialization.format'='\'')
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+#### A masked pattern was here ####
+TBLPROPERTIES (
+ 'bucketing_version'='2',
+#### A masked pattern was here ####
+PREHOOK: query: drop table esc_special_delimiter
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@esc_special_delimiter
+PREHOOK: Output: default@esc_special_delimiter
+POSTHOOK: query: drop table esc_special_delimiter
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@esc_special_delimiter
+POSTHOOK: Output: default@esc_special_delimiter
+PREHOOK: query: create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';' stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@esc_special_delimiter
+POSTHOOK: query: create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';' stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@esc_special_delimiter
+PREHOOK: query: show create table esc_special_delimiter
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: default@esc_special_delimiter
+POSTHOOK: query: show create table esc_special_delimiter
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: default@esc_special_delimiter
+CREATE TABLE `esc_special_delimiter`(
+ `age` int,
+ `name` string)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+WITH SERDEPROPERTIES (
+ 'field.delim'='\;',
+ 'serialization.format'='\;')
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+#### A masked pattern was here ####
+TBLPROPERTIES (
+ 'bucketing_version'='2',
+#### A masked pattern was here ####
+PREHOOK: query: drop table esc_special_delimiter
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@esc_special_delimiter
+PREHOOK: Output: default@esc_special_delimiter
+POSTHOOK: query: drop table esc_special_delimiter
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@esc_special_delimiter
+POSTHOOK: Output: default@esc_special_delimiter