You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dk...@apache.org on 2023/08/17 10:36:10 UTC
[hive] branch master updated: HIVE-27237: Iceberg: DROP TAG SQL implementation (Butao Zhang, reviewed by Denys Kuzmenko)
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new eb364fd11c7 HIVE-27237: Iceberg: DROP TAG SQL implementation (Butao Zhang, reviewed by Denys Kuzmenko)
eb364fd11c7 is described below
commit eb364fd11c7cc04988e63a040c1d0bbd89bea0c5
Author: Butao Zhang <zh...@cmss.chinamobile.com>
AuthorDate: Thu Aug 17 18:36:04 2023 +0800
HIVE-27237: Iceberg: DROP TAG SQL implementation (Butao Zhang, reviewed by Denys Kuzmenko)
Closes #4551
---
.../iceberg/mr/hive/HiveIcebergMetaHook.java | 2 +-
.../iceberg/mr/hive/HiveIcebergStorageHandler.java | 5 +++
.../org/apache/iceberg/mr/hive/IcebergTagExec.java | 12 ++++++
.../mr/hive/TestHiveIcebergTagOperation.java | 26 ++++++++++++
.../test/queries/positive/alter_table_create_tag.q | 12 ++++++
.../results/positive/alter_table_create_tag.q.out | 48 ++++++++++++++++++++++
.../hadoop/hive/ql/parse/AlterClauseParser.g | 10 ++++-
.../org/apache/hadoop/hive/ql/parse/HiveParser.g | 1 +
.../hadoop/hive/ql/ddl/table/AlterTableType.java | 1 +
.../tag/drop/AlterTableDropTagAnalyzer.java | 35 ++++++++++++++++
.../apache/hadoop/hive/ql/plan/HiveOperation.java | 1 +
.../authorization/plugin/HiveOperationType.java | 1 +
.../plugin/sqlstd/Operation2Privilege.java | 2 +
13 files changed, 154 insertions(+), 2 deletions(-)
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
index b7900b97e11..33f2743483f 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
@@ -121,7 +121,7 @@ public class HiveIcebergMetaHook implements HiveMetaHook {
AlterTableType.ADDCOLS, AlterTableType.REPLACE_COLUMNS, AlterTableType.RENAME_COLUMN,
AlterTableType.ADDPROPS, AlterTableType.DROPPROPS, AlterTableType.SETPARTITIONSPEC,
AlterTableType.UPDATE_COLUMNS, AlterTableType.RENAME, AlterTableType.EXECUTE, AlterTableType.CREATE_BRANCH,
- AlterTableType.CREATE_TAG, AlterTableType.DROP_BRANCH);
+ AlterTableType.CREATE_TAG, AlterTableType.DROP_BRANCH, AlterTableType.DROP_TAG);
private static final List<String> MIGRATION_ALLOWED_SOURCE_FORMATS = ImmutableList.of(
FileFormat.PARQUET.name().toLowerCase(),
FileFormat.ORC.name().toLowerCase(),
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index 6737a6bff8b..ef74cf6bf31 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -871,6 +871,11 @@ public class HiveIcebergStorageHandler implements HiveStoragePredicateHandler, H
(AlterTableSnapshotRefSpec.DropSnapshotRefSpec) alterTableSnapshotRefSpec.getOperationParams();
IcebergBranchExec.dropBranch(icebergTable, dropBranchSpec);
break;
+ case DROP_TAG:
+ AlterTableSnapshotRefSpec.DropSnapshotRefSpec dropTagSpec =
+ (AlterTableSnapshotRefSpec.DropSnapshotRefSpec) alterTableSnapshotRefSpec.getOperationParams();
+ IcebergTagExec.dropTag(icebergTable, dropTagSpec);
+ break;
default:
throw new UnsupportedOperationException(String.format(
"Operation type %s is not supported", alterTableSnapshotRefSpec.getOperationType().getName()));
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTagExec.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTagExec.java
index 705d152bb34..54e386af225 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTagExec.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTagExec.java
@@ -21,6 +21,7 @@ package org.apache.iceberg.mr.hive;
import org.apache.hadoop.hive.ql.parse.AlterTableSnapshotRefSpec;
import org.apache.iceberg.ManageSnapshots;
+import org.apache.iceberg.SnapshotRef;
import org.apache.iceberg.Table;
import org.apache.iceberg.util.SnapshotUtil;
import org.slf4j.Logger;
@@ -52,4 +53,15 @@ public class IcebergTagExec {
manageSnapshots.commit();
}
+
+ public static void dropTag(Table table, AlterTableSnapshotRefSpec.DropSnapshotRefSpec dropTagSpec) {
+ String tagName = dropTagSpec.getRefName();
+ boolean ifExists = dropTagSpec.getIfExists();
+
+ SnapshotRef snapshotRef = table.refs().get(tagName);
+ if (snapshotRef != null || !ifExists) {
+ LOG.info("Dropping tag {} on iceberg table {}", tagName, table.name());
+ table.manageSnapshots().removeTag(tagName).commit();
+ }
+ }
}
diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergTagOperation.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergTagOperation.java
index 8e7148c8609..3919328fd26 100644
--- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergTagOperation.java
+++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergTagOperation.java
@@ -174,4 +174,30 @@ public class TestHiveIcebergTagOperation extends HiveIcebergStorageHandlerWithEn
Assert.assertTrue(e.getMessage().contains("Don't support write (insert/delete/update/merge) to iceberg tag"));
}
}
+
+ @Test
+ public void testDropTag() throws InterruptedException, IOException {
+ Table table =
+ testTables.createTableWithVersions(shell, "customers", HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA,
+ fileFormat, HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, 2);
+
+ String tagName = "test_tag_1";
+ shell.executeStatement(String.format("ALTER TABLE customers CREATE TAG %s", tagName));
+ table.refresh();
+ Assert.assertNotNull(table.refs().get(tagName));
+
+ shell.executeStatement(String.format("ALTER TABLE customers DROP TAG IF EXISTS %s", tagName));
+ table.refresh();
+ Assert.assertNull(table.refs().get(tagName));
+
+ try {
+ // drop a non-exist tag
+ shell.executeStatement(String.format("ALTER TABLE customers DROP TAG %s", tagName));
+ } catch (Throwable e) {
+ while (e.getCause() != null) {
+ e = e.getCause();
+ }
+ Assert.assertTrue(e.getMessage().contains("Tag does not exist: test_tag_1"));
+ }
+ }
}
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/alter_table_create_tag.q b/iceberg/iceberg-handler/src/test/queries/positive/alter_table_create_tag.q
index dbbde623d84..ad5eabb3e8b 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/alter_table_create_tag.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/alter_table_create_tag.q
@@ -17,3 +17,15 @@ explain alter table iceTbl create tag test_tag_2 retain 5 days;
alter table iceTbl create tag test_tag_2 retain 5 days;
select name, max_reference_age_in_ms from default.iceTbl.refs where type='TAG';
+-- drop a tag
+explain alter table iceTbl drop tag test_tag_2;
+alter table iceTbl drop tag test_tag_2;
+
+-- drop a tag with if exists
+explain alter table iceTbl drop tag if exists test_tag_3;
+alter table iceTbl drop tag if exists test_tag_3;
+
+-- drop a non-exist tag with if exists
+alter table iceTbl drop tag if exists test_tag_4;
+
+
diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_table_create_tag.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_table_create_tag.q.out
index 66b3eaa705a..312a93f7f05 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/alter_table_create_tag.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/alter_table_create_tag.q.out
@@ -83,3 +83,51 @@ POSTHOOK: Input: default@icetbl
POSTHOOK: Output: hdfs://### HDFS PATH ###
test_tag_1 NULL
test_tag_2 432000000
+PREHOOK: query: explain alter table iceTbl drop tag test_tag_2
+PREHOOK: type: ALTERTABLE_DROPTAG
+PREHOOK: Input: default@icetbl
+POSTHOOK: query: explain alter table iceTbl drop tag test_tag_2
+POSTHOOK: type: ALTERTABLE_DROPTAG
+POSTHOOK: Input: default@icetbl
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ SnapshotRef Operation
+ table name: default.iceTbl
+ spec: AlterTableSnapshotRefSpec{operationType=DROP_TAG, operationParams=DropSnapshotRefSpec{refName=test_tag_2, ifExists=false}}
+
+PREHOOK: query: alter table iceTbl drop tag test_tag_2
+PREHOOK: type: ALTERTABLE_DROPTAG
+PREHOOK: Input: default@icetbl
+POSTHOOK: query: alter table iceTbl drop tag test_tag_2
+POSTHOOK: type: ALTERTABLE_DROPTAG
+POSTHOOK: Input: default@icetbl
+PREHOOK: query: explain alter table iceTbl drop tag if exists test_tag_3
+PREHOOK: type: ALTERTABLE_DROPTAG
+PREHOOK: Input: default@icetbl
+POSTHOOK: query: explain alter table iceTbl drop tag if exists test_tag_3
+POSTHOOK: type: ALTERTABLE_DROPTAG
+POSTHOOK: Input: default@icetbl
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ SnapshotRef Operation
+ table name: default.iceTbl
+ spec: AlterTableSnapshotRefSpec{operationType=DROP_TAG, operationParams=DropSnapshotRefSpec{refName=test_tag_3, ifExists=true}}
+
+PREHOOK: query: alter table iceTbl drop tag if exists test_tag_3
+PREHOOK: type: ALTERTABLE_DROPTAG
+PREHOOK: Input: default@icetbl
+POSTHOOK: query: alter table iceTbl drop tag if exists test_tag_3
+POSTHOOK: type: ALTERTABLE_DROPTAG
+POSTHOOK: Input: default@icetbl
+PREHOOK: query: alter table iceTbl drop tag if exists test_tag_4
+PREHOOK: type: ALTERTABLE_DROPTAG
+PREHOOK: Input: default@icetbl
+POSTHOOK: query: alter table iceTbl drop tag if exists test_tag_4
+POSTHOOK: type: ALTERTABLE_DROPTAG
+POSTHOOK: Input: default@icetbl
diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g
index aabc136bf5c..ddb2285aafd 100644
--- a/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g
+++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g
@@ -75,9 +75,10 @@ alterTableStatementSuffix
| alterStatementSuffixSetPartSpec
| alterStatementSuffixExecute
| alterStatementSuffixCreateBranch
+ | alterStatementSuffixDropBranch
| alterStatementSuffixCreateTag
+ | alterStatementSuffixDropTag
| alterStatementSuffixConvert
- | alterStatementSuffixDropBranch
;
alterTblPartitionStatementSuffix[boolean partition]
@@ -518,6 +519,13 @@ retentionOfSnapshots
-> ^(TOK_WITH_SNAPSHOT_RETENTION $minSnapshotsToKeep ($maxSnapshotAge $timeUnit)?)
;
+alterStatementSuffixDropTag
+@init { gParent.pushMsg("alter table drop tag (if exists) tagName", state); }
+@after { gParent.popMsg(state); }
+ : KW_DROP KW_TAG ifExists? tagName=identifier
+ -> ^(TOK_ALTERTABLE_DROP_TAG ifExists? $tagName)
+ ;
+
alterStatementSuffixCreateTag
@init { gParent.pushMsg("alter table create tag", state); }
@after { gParent.popMsg(state); }
diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index 1fb963e297a..d8a8ce8b27d 100644
--- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -222,6 +222,7 @@ TOK_ALTERTABLE_EXECUTE;
TOK_ALTERTABLE_CREATE_BRANCH;
TOK_ALTERTABLE_DROP_BRANCH;
TOK_ALTERTABLE_CREATE_TAG;
+TOK_ALTERTABLE_DROP_TAG;
TOK_RETAIN;
TOK_WITH_SNAPSHOT_RETENTION;
TOK_ALTERTABLE_CONVERT;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java
index a0e59181283..7da882c641b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java
@@ -43,6 +43,7 @@ public enum AlterTableType {
CREATE_BRANCH("create branch"),
DROP_BRANCH("drop branch"),
CREATE_TAG("create tag"),
+ DROP_TAG("drop tag"),
// constraint
ADD_CONSTRAINT("add constraint"),
DROP_CONSTRAINT("drop constraint"),
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/tag/drop/AlterTableDropTagAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/tag/drop/AlterTableDropTagAnalyzer.java
new file mode 100644
index 00000000000..acd9688118e
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/tag/drop/AlterTableDropTagAnalyzer.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table.snapshotref.tag.drop;
+
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
+import org.apache.hadoop.hive.ql.ddl.table.snapshotref.AlterTableDropSnapshotRefAnalyzer;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+
+@DDLSemanticAnalyzerFactory.DDLType(types = HiveParser.TOK_ALTERTABLE_DROP_TAG)
+public class AlterTableDropTagAnalyzer extends AlterTableDropSnapshotRefAnalyzer {
+
+ public AlterTableDropTagAnalyzer(QueryState queryState) throws SemanticException {
+ super(queryState);
+ alterTableType = AlterTableType.DROP_TAG;
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java
index 9e9668742e9..020453d682b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java
@@ -80,6 +80,7 @@ public enum HiveOperation {
ALTERTABLE_CREATEBRANCH("ALTERTABLE_CREATEBRANCH", HiveParser.TOK_ALTERTABLE_CREATE_BRANCH, null, null),
ALTERTABLE_CREATETAG("ALTERTABLE_CREATETAG", HiveParser.TOK_ALTERTABLE_CREATE_TAG, null, null),
ALTERTABLE_DROPBRANCH("ALTERTABLE_DROPBRANCH", HiveParser.TOK_ALTERTABLE_DROP_BRANCH, null, null),
+ ALTERTABLE_DROPTAG("ALTERTABLE_DROPTAG", HiveParser.TOK_ALTERTABLE_DROP_TAG, null, null),
ALTERTABLE_CONVERT("ALTERTABLE_CONVERT", HiveParser.TOK_ALTERTABLE_CONVERT, null, null),
ALTERTABLE_SERIALIZER("ALTERTABLE_SERIALIZER", HiveParser.TOK_ALTERTABLE_SERIALIZER,
new Privilege[]{Privilege.ALTER_METADATA}, null),
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java
index 447a0bcf4b2..68d008eeda3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java
@@ -142,6 +142,7 @@ public enum HiveOperationType {
ALTERTABLE_CREATEBRANCH,
ALTERTABLE_DROPBRANCH,
ALTERTABLE_CREATETAG,
+ ALTERTABLE_DROPTAG,
SHOW_COMPACTIONS,
SHOW_TRANSACTIONS,
ABORT_TRANSACTIONS,
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java
index 97e08a29c23..4cb7a57cade 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java
@@ -246,6 +246,8 @@ public class Operation2Privilege {
PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR));
op2Priv.put(HiveOperationType.ALTERTABLE_DROPBRANCH,
PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR));
+ op2Priv.put(HiveOperationType.ALTERTABLE_DROPTAG,
+ PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR));
// require view ownership for alter/drop view
op2Priv.put(HiveOperationType.ALTERVIEW_PROPERTIES,