You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2020/02/29 07:20:54 UTC
[hive] branch master updated: HIVE-22920 : Add row format
OpenCSVSerde to the metastore column managed list (Ramesh Kumar via
Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
hashutosh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new a12ebb8 HIVE-22920 : Add row format OpenCSVSerde to the metastore column managed list (Ramesh Kumar via Ashutosh Chauhan)
a12ebb8 is described below
commit a12ebb8481f2e004cd03431e5f2ba691dc20abcc
Author: Ramesh Kumar Thangarajan <ra...@cloudera.com>
AuthorDate: Fri Feb 28 23:18:17 2020 -0800
HIVE-22920 : Add row format OpenCSVSerde to the metastore column managed list (Ramesh Kumar via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 3 +-
.../test/resources/testconfiguration.properties | 1 +
.../org/apache/hadoop/hive/ql/metadata/Table.java | 4 +-
ql/src/test/queries/clientpositive/alter6.q | 4 ++
.../test/results/clientpositive/llap/alter6.q.out | 71 ++++++++++++++++++++++
.../hadoop/hive/metastore/conf/MetastoreConf.java | 3 +-
6 files changed, 83 insertions(+), 3 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 0eda0d2..62aa4f0 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1869,7 +1869,8 @@ public class HiveConf extends Configuration {
"org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe," +
"org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe," +
"org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe," +
- "org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe",
+ "org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe," +
+ "org.apache.hadoop.hive.serde2.OpenCSVSerde",
"SerDes retrieving schema from metastore. This is an internal parameter."),
@Deprecated
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 8ee9bd6..743fe2c 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -460,6 +460,7 @@ minillaplocal.query.files=\
acid_vectorization_original.q,\
alter_merge_stats_orc.q,\
alter_partition_change_col.q,\
+ alter6.q,\
authorization_view_8.q,\
auto_join30.q,\
auto_join_filters.q,\
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
index b9bb3ab..6d56536 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.HiveMetaStoreUtils;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
@@ -1064,7 +1065,8 @@ public class Table implements Serializable {
public static boolean hasMetastoreBasedSchema(HiveConf conf, String serdeLib) {
return StringUtils.isEmpty(serdeLib) ||
- conf.getStringCollection(ConfVars.SERDESUSINGMETASTOREFORSCHEMA.varname).contains(serdeLib);
+ MetastoreConf.getStringCollection(conf,
+ MetastoreConf.ConfVars.SERDES_USING_METASTORE_FOR_SCHEMA).contains(serdeLib);
}
public static boolean shouldStoreFieldsInMetastore(
diff --git a/ql/src/test/queries/clientpositive/alter6.q b/ql/src/test/queries/clientpositive/alter6.q
new file mode 100644
index 0000000..f076356
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/alter6.q
@@ -0,0 +1,4 @@
+CREATE TABLE t3_br_eod_custody(data_publication_date_time string COMMENT 'from deserializer', booking_entity_name string COMMENT 'from deserializer', booking_entity_country_iso_code string COMMENT 'from deserializer', booking_entity_trade_portfolio_name string COMMENT 'from deserializer', isin string COMMENT 'from deserializer', coupon_rate string COMMENT 'from deserializer', instrument_description string COMMENT 'from deserializer', maturity_date string COMMENT 'from deserializer', sour [...]
+alter table t3_br_eod_custody add columns (HKMA_Instrument_Id string) CASCADE;
+select * from t3_br_eod_custody;
+show create table t3_br_eod_custody;
diff --git a/ql/src/test/results/clientpositive/llap/alter6.q.out b/ql/src/test/results/clientpositive/llap/alter6.q.out
new file mode 100644
index 0000000..4d6c59e
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/alter6.q.out
@@ -0,0 +1,71 @@
+PREHOOK: query: CREATE TABLE t3_br_eod_custody(data_publication_date_time string COMMENT 'from deserializer', booking_entity_name string COMMENT 'from deserializer', booking_entity_country_iso_code string COMMENT 'from deserializer', booking_entity_trade_portfolio_name string COMMENT 'from deserializer', isin string COMMENT 'from deserializer', coupon_rate string COMMENT 'from deserializer', instrument_description string COMMENT 'from deserializer', maturity_date string COMMENT 'from des [...]
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t3_br_eod_custody
+POSTHOOK: query: CREATE TABLE t3_br_eod_custody(data_publication_date_time string COMMENT 'from deserializer', booking_entity_name string COMMENT 'from deserializer', booking_entity_country_iso_code string COMMENT 'from deserializer', booking_entity_trade_portfolio_name string COMMENT 'from deserializer', isin string COMMENT 'from deserializer', coupon_rate string COMMENT 'from deserializer', instrument_description string COMMENT 'from deserializer', maturity_date string COMMENT 'from de [...]
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t3_br_eod_custody
+PREHOOK: query: alter table t3_br_eod_custody add columns (HKMA_Instrument_Id string) CASCADE
+PREHOOK: type: ALTERTABLE_ADDCOLS
+PREHOOK: Input: default@t3_br_eod_custody
+PREHOOK: Output: default@t3_br_eod_custody
+POSTHOOK: query: alter table t3_br_eod_custody add columns (HKMA_Instrument_Id string) CASCADE
+POSTHOOK: type: ALTERTABLE_ADDCOLS
+POSTHOOK: Input: default@t3_br_eod_custody
+POSTHOOK: Output: default@t3_br_eod_custody
+PREHOOK: query: select * from t3_br_eod_custody
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t3_br_eod_custody
+#### A masked pattern was here ####
+POSTHOOK: query: select * from t3_br_eod_custody
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t3_br_eod_custody
+#### A masked pattern was here ####
+PREHOOK: query: show create table t3_br_eod_custody
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: default@t3_br_eod_custody
+POSTHOOK: query: show create table t3_br_eod_custody
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: default@t3_br_eod_custody
+CREATE TABLE `t3_br_eod_custody`(
+ `data_publication_date_time` string COMMENT 'from deserializer',
+ `booking_entity_name` string COMMENT 'from deserializer',
+ `booking_entity_country_iso_code` string COMMENT 'from deserializer',
+ `booking_entity_trade_portfolio_name` string COMMENT 'from deserializer',
+ `isin` string COMMENT 'from deserializer',
+ `coupon_rate` string COMMENT 'from deserializer',
+ `instrument_description` string COMMENT 'from deserializer',
+ `maturity_date` string COMMENT 'from deserializer',
+ `source_system_instrument_type` string COMMENT 'from deserializer',
+ `source_system_instrument_sub_type` string COMMENT 'from deserializer',
+ `settlement_currency` string COMMENT 'from deserializer',
+ `quantity` string COMMENT 'from deserializer',
+ `parvalue` string COMMENT 'from deserializer',
+ `ifrs9_instrument_classification` string COMMENT 'from deserializer',
+ `outstanding_amount` string COMMENT 'from deserializer',
+ `principal_repayment_factor` string COMMENT 'from deserializer',
+ `day_count_fraction` string COMMENT 'from deserializer',
+ `cusip` string COMMENT 'from deserializer',
+ `figi` string COMMENT 'from deserializer',
+ `issuer_name` string COMMENT 'from deserializer',
+ `notional_amount_currency` string COMMENT 'from deserializer',
+ `settlement_date_of_near_transaction` string COMMENT 'from deserializer',
+ `custodian_account_id` string COMMENT 'from deserializer',
+ `hkma_instrument_id` string)
+PARTITIONED BY (
+ `reporttype` string,
+ `source` string,
+ `eodlevel` string,
+ `currentbpd` string)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+#### A masked pattern was here ####
+TBLPROPERTIES (
+ 'bucketing_version'='2',
+#### A masked pattern was here ####
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 58b67e8..fc6a2fd 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -962,7 +962,8 @@ public class MetastoreConf {
"org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe," +
"org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe," +
"org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe," +
- "org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe",
+ "org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe," +
+ "org.apache.hadoop.hive.serde2.OpenCSVSerde",
"SerDes retrieving schema from metastore. This is an internal parameter."),
SERVER_MAX_MESSAGE_SIZE("metastore.server.max.message.size",
"hive.metastore.server.max.message.size", 100*1024*1024L,