You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by lp...@apache.org on 2022/07/29 08:58:08 UTC
[hive] branch master updated: HIVE-26421: HMSClient atler_table_req() is called twice when running an alter operation on iceberg table (#3469) (Laszlo Pinter, reviewed by Adam Szita)
This is an automated email from the ASF dual-hosted git repository.
lpinter pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8863468898c HIVE-26421: HMSClient atler_table_req() is called twice when running an alter operation on iceberg table (#3469) (Laszlo Pinter, reviewed by Adam Szita)
8863468898c is described below
commit 8863468898c9242fc334b9c93bf79e249e53842d
Author: László Pintér <47...@users.noreply.github.com>
AuthorDate: Fri Jul 29 10:57:57 2022 +0200
HIVE-26421: HMSClient atler_table_req() is called twice when running an alter operation on iceberg table (#3469) (Laszlo Pinter, reviewed by Adam Szita)
---
.../iceberg/mr/hive/HiveIcebergMetaHook.java | 6 ++-
.../apache/hadoop/hive/metastore/HiveMetaHook.java | 2 +
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 24 ++++++-----
.../hive/metastore/utils/MetaStoreUtils.java | 46 +++++++++++-----------
4 files changed, 46 insertions(+), 32 deletions(-)
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
index 1191ca8e431..f3572673742 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
@@ -268,8 +268,8 @@ public class HiveIcebergMetaHook implements HiveMetaHook {
@Override
public void preAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, EnvironmentContext context)
throws MetaException {
- setupAlterOperationType(hmsTable, context);
catalogProperties = getCatalogProperties(hmsTable);
+ setupAlterOperationType(hmsTable, context);
try {
icebergTable = IcebergTableUtil.getTable(conf, catalogProperties);
} catch (NoSuchTableException nte) {
@@ -500,6 +500,10 @@ public class HiveIcebergMetaHook implements HiveMetaHook {
"Unsupported ALTER TABLE operation type on Iceberg table " + tableName + ", must be one of: " +
SUPPORTED_ALTER_OPS);
}
+
+ if (currentAlterTableOp != AlterTableType.ADDPROPS && Catalogs.hiveCatalog(conf, catalogProperties)) {
+ context.getProperties().put(SKIP_METASTORE_ALTER, "true");
+ }
}
}
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
index b2e15ee9a66..3accd3cd9ea 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
@@ -52,6 +52,8 @@ public interface HiveMetaHook {
String PROPERTIES_SEPARATOR = "'";
String MIGRATE_HIVE_TO_ICEBERG = "migrate_hive_to_iceberg";
String INITIALIZE_ROLLBACK_MIGRATION = "initialize_rollback_migration";
+ // if this flag is set to true, the HMS call from HiveMetaStoreClient#alter_table() will be skipped
+ String SKIP_METASTORE_ALTER = "skip_metastore_alter";
/**
* Called before a new table definition is added to the metastore
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index ecf85fb817a..ab6a1aa9607 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -562,17 +562,23 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
if (hook != null) {
hook.preAlterTable(new_tbl, envContext);
}
- AlterTableRequest req = new AlterTableRequest(dbName, tbl_name, new_tbl);
- req.setCatName(catName);
- req.setValidWriteIdList(validWriteIds);
- req.setEnvironmentContext(envContext);
- if (processorCapabilities != null) {
- req.setProcessorCapabilities(new ArrayList<String>(Arrays.asList(processorCapabilities)));
- req.setProcessorIdentifier(processorIdentifier);
- }
boolean success = false;
try {
- client.alter_table_req(req);
+ boolean skipAlter = envContext != null && envContext.getProperties() != null &&
+ Boolean.valueOf(envContext.getProperties().getOrDefault(HiveMetaHook.SKIP_METASTORE_ALTER, "false"));
+ if (!skipAlter) {
+ AlterTableRequest req = new AlterTableRequest(dbName, tbl_name, new_tbl);
+ req.setCatName(catName);
+ req.setValidWriteIdList(validWriteIds);
+ req.setEnvironmentContext(envContext);
+ if (processorCapabilities != null) {
+ req.setProcessorCapabilities(new ArrayList<String>(Arrays.asList(processorCapabilities)));
+ req.setProcessorIdentifier(processorIdentifier);
+ }
+
+ client.alter_table_req(req);
+ }
+
if (hook != null) {
hook.commitAlterTable(new_tbl, envContext);
}
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
index 6e1a8ef370e..d41310906a7 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
@@ -799,29 +799,31 @@ public class MetaStoreUtils {
}
}
- String partString = StringUtils.EMPTY;
- String partStringSep = StringUtils.EMPTY;
- String partTypesString = StringUtils.EMPTY;
- String partTypesStringSep = StringUtils.EMPTY;
- for (FieldSchema partKey : partitionKeys) {
- partString = partString.concat(partStringSep);
- partString = partString.concat(partKey.getName());
- partTypesString = partTypesString.concat(partTypesStringSep);
- partTypesString = partTypesString.concat(partKey.getType());
- if (partStringSep.length() == 0) {
- partStringSep = "/";
- partTypesStringSep = ":";
+ if (partitionKeys != null) {
+ String partString = StringUtils.EMPTY;
+ String partStringSep = StringUtils.EMPTY;
+ String partTypesString = StringUtils.EMPTY;
+ String partTypesStringSep = StringUtils.EMPTY;
+ for (FieldSchema partKey : partitionKeys) {
+ partString = partString.concat(partStringSep);
+ partString = partString.concat(partKey.getName());
+ partTypesString = partTypesString.concat(partTypesStringSep);
+ partTypesString = partTypesString.concat(partKey.getType());
+ if (partStringSep.length() == 0) {
+ partStringSep = "/";
+ partTypesStringSep = ":";
+ }
+ }
+ if (partString.length() > 0) {
+ schema
+ .setProperty(
+ org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS,
+ partString);
+ schema
+ .setProperty(
+ org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES,
+ partTypesString);
}
- }
- if (partString.length() > 0) {
- schema
- .setProperty(
- org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS,
- partString);
- schema
- .setProperty(
- org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES,
- partTypesString);
}
if (parameters != null) {