You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ng...@apache.org on 2023/10/04 16:34:03 UTC
[hive] branch master updated: HIVE-27728: Renaming partition from legacy tables should move the partition to the new location (#4752)
This is an automated email from the ASF dual-hosted git repository.
ngangam pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5c73a847cd3 HIVE-27728: Renaming partition from legacy tables should move the partition to the new location (#4752)
5c73a847cd3 is described below
commit 5c73a847cd385424cdc8a7804176fbae7834c90f
Author: dengzh <de...@gmail.com>
AuthorDate: Thu Oct 5 00:33:53 2023 +0800
HIVE-27728: Renaming partition from legacy tables should move the partition to the new location (#4752)
---
.../clientpositive/translated_external_rename5.q | 26 ++++
.../llap/translated_external_rename5.q.out | 160 +++++++++++++++++++++
.../hadoop/hive/metastore/HiveAlterHandler.java | 4 +-
3 files changed, 189 insertions(+), 1 deletion(-)
diff --git a/ql/src/test/queries/clientpositive/translated_external_rename5.q b/ql/src/test/queries/clientpositive/translated_external_rename5.q
new file mode 100644
index 00000000000..033de9b30ab
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/translated_external_rename5.q
@@ -0,0 +1,26 @@
+--! qt:replace:/^(?!LOCATION|.*HOOK).*metadata_table_test1/### TABLE DIRECTORY ###/
+
+set metastore.metadata.transformer.class=org.apache.hadoop.hive.metastore.MetastoreDefaultTransformer;
+set hive.create.as.external.legacy=true;
+
+CREATE TABLE metadata_test1(
+ emp_number int,
+ emp_name string,
+ city string)
+PARTITIONED BY(state string)
+LOCATION 'pfile://${system:test.tmp.dir}/metadata_table_test1';
+
+DESC FORMATTED metadata_test1;
+
+INSERT INTO metadata_test1 PARTITION(state='A') VALUES (11, 'ABC', 'AA');
+INSERT INTO metadata_test1 PARTITION(state='B') VALUES (12, 'XYZ', 'BX'), (13, 'UVW', 'BU');
+select * from default.metadata_test1;
+select "======================== list table directory =========================";
+dfs -ls ${system:test.tmp.dir}/metadata_table_test1/;
+
+ALTER TABLE default.metadata_test1 PARTITION (state='A') RENAME TO PARTITION (state='C');
+select * from metadata_test1;
+select "======================== list table directory =========================";
+dfs -ls ${system:test.tmp.dir}/metadata_table_test1/;
+
+drop table metadata_test1;
diff --git a/ql/src/test/results/clientpositive/llap/translated_external_rename5.q.out b/ql/src/test/results/clientpositive/llap/translated_external_rename5.q.out
new file mode 100644
index 00000000000..5519b54c79a
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/translated_external_rename5.q.out
@@ -0,0 +1,160 @@
+PREHOOK: query: CREATE TABLE metadata_test1(
+ emp_number int,
+ emp_name string,
+ city string)
+PARTITIONED BY(state string)
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@metadata_test1
+POSTHOOK: query: CREATE TABLE metadata_test1(
+ emp_number int,
+ emp_name string,
+ city string)
+PARTITIONED BY(state string)
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@metadata_test1
+PREHOOK: query: DESC FORMATTED metadata_test1
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@metadata_test1
+POSTHOOK: query: DESC FORMATTED metadata_test1
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@metadata_test1
+# col_name data_type comment
+emp_number int
+emp_name string
+city string
+
+# Partition Information
+# col_name data_type comment
+state string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+### TABLE DIRECTORY ###
+Table Type: EXTERNAL_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ EXTERNAL TRUE
+ TRANSLATED_TO_EXTERNAL TRUE
+ bucketing_version 2
+ external.table.purge TRUE
+ numFiles 0
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ totalSize 0
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: INSERT INTO metadata_test1 PARTITION(state='A') VALUES (11, 'ABC', 'AA')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@metadata_test1@state=A
+POSTHOOK: query: INSERT INTO metadata_test1 PARTITION(state='A') VALUES (11, 'ABC', 'AA')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@metadata_test1@state=A
+POSTHOOK: Lineage: metadata_test1 PARTITION(state=A).city SCRIPT []
+POSTHOOK: Lineage: metadata_test1 PARTITION(state=A).emp_name SCRIPT []
+POSTHOOK: Lineage: metadata_test1 PARTITION(state=A).emp_number SCRIPT []
+PREHOOK: query: INSERT INTO metadata_test1 PARTITION(state='B') VALUES (12, 'XYZ', 'BX'), (13, 'UVW', 'BU')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@metadata_test1@state=B
+POSTHOOK: query: INSERT INTO metadata_test1 PARTITION(state='B') VALUES (12, 'XYZ', 'BX'), (13, 'UVW', 'BU')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@metadata_test1@state=B
+POSTHOOK: Lineage: metadata_test1 PARTITION(state=B).city SCRIPT []
+POSTHOOK: Lineage: metadata_test1 PARTITION(state=B).emp_name SCRIPT []
+POSTHOOK: Lineage: metadata_test1 PARTITION(state=B).emp_number SCRIPT []
+PREHOOK: query: select * from default.metadata_test1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@metadata_test1
+PREHOOK: Input: default@metadata_test1@state=A
+PREHOOK: Input: default@metadata_test1@state=B
+#### A masked pattern was here ####
+POSTHOOK: query: select * from default.metadata_test1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@metadata_test1
+POSTHOOK: Input: default@metadata_test1@state=A
+POSTHOOK: Input: default@metadata_test1@state=B
+#### A masked pattern was here ####
+11 ABC AA A
+12 XYZ BX B
+13 UVW BU B
+PREHOOK: query: select "======================== list table directory ========================="
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select "======================== list table directory ========================="
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+======================== list table directory =========================
+Found 2 items
+### TABLE DIRECTORY ###/state=A
+### TABLE DIRECTORY ###/state=B
+PREHOOK: query: ALTER TABLE default.metadata_test1 PARTITION (state='A') RENAME TO PARTITION (state='C')
+PREHOOK: type: ALTERTABLE_RENAMEPART
+PREHOOK: Input: default@metadata_test1
+PREHOOK: Output: default@metadata_test1@state=A
+POSTHOOK: query: ALTER TABLE default.metadata_test1 PARTITION (state='A') RENAME TO PARTITION (state='C')
+POSTHOOK: type: ALTERTABLE_RENAMEPART
+POSTHOOK: Input: default@metadata_test1
+POSTHOOK: Input: default@metadata_test1@state=A
+POSTHOOK: Output: default@metadata_test1@state=A
+POSTHOOK: Output: default@metadata_test1@state=C
+PREHOOK: query: select * from metadata_test1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@metadata_test1
+PREHOOK: Input: default@metadata_test1@state=B
+PREHOOK: Input: default@metadata_test1@state=C
+#### A masked pattern was here ####
+POSTHOOK: query: select * from metadata_test1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@metadata_test1
+POSTHOOK: Input: default@metadata_test1@state=B
+POSTHOOK: Input: default@metadata_test1@state=C
+#### A masked pattern was here ####
+12 XYZ BX B
+13 UVW BU B
+11 ABC AA C
+PREHOOK: query: select "======================== list table directory ========================="
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select "======================== list table directory ========================="
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+======================== list table directory =========================
+Found 2 items
+### TABLE DIRECTORY ###/state=B
+### TABLE DIRECTORY ###/state=C
+PREHOOK: query: drop table metadata_test1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@metadata_test1
+PREHOOK: Output: database:default
+PREHOOK: Output: default@metadata_test1
+POSTHOOK: query: drop table metadata_test1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@metadata_test1
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@metadata_test1
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 571f5aae361..9bce99ad680 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -679,7 +679,9 @@ public class HiveAlterHandler implements AlterHandler {
// 1) partition SD Location
// 2) partition column stats if there are any because of part_name field in HMS table PART_COL_STATS
// 3) rename the partition directory if it is not an external table
- if (!tbl.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
+ boolean shouldMoveData = !(MetaStoreUtils.isExternalTable(tbl) &&
+ !MetaStoreUtils.isPropertyTrue(tbl.getParameters(), "TRANSLATED_TO_EXTERNAL"));
+ if (shouldMoveData) {
// TODO: refactor this into a separate method after master merge, this one is too big.
try {
db = msdb.getDatabase(catName, dbname);