You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2022/04/01 11:51:46 UTC
[hive] branch master updated: HIVE-26101: Port Iceberg Hive fix - Hive: Avoid recursive listing in HiveCatalog#renameTable (Peter Vary reviewed by Marton Bod) (#3163)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4641f6e HIVE-26101: Port Iceberg Hive fix - Hive: Avoid recursive listing in HiveCatalog#renameTable (Peter Vary reviewed by Marton Bod) (#3163)
4641f6e is described below
commit 4641f6e484915cb8dfa5abedfc4732b68c88a758
Author: pvary <pv...@cloudera.com>
AuthorDate: Fri Apr 1 13:51:27 2022 +0200
HIVE-26101: Port Iceberg Hive fix - Hive: Avoid recursive listing in HiveCatalog#renameTable (Peter Vary reviewed by Marton Bod) (#3163)
---
.../java/org/apache/iceberg/hive/HiveCatalog.java | 2 +-
.../apache/iceberg/hive/HiveTableOperations.java | 16 ++-----------
.../org/apache/iceberg/hive/MetastoreUtil.java | 27 ++++++++++++++++++++++
3 files changed, 30 insertions(+), 15 deletions(-)
diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveCatalog.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
index 880d60d..4737dd6 100644
--- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
+++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
@@ -214,7 +214,7 @@ public class HiveCatalog extends BaseMetastoreCatalog implements SupportsNamespa
table.setTableName(to.name());
clients.run(client -> {
- client.alter_table(fromDatabase, fromName, table);
+ MetastoreUtil.alterTable(client, fromDatabase, fromName, table);
return null;
});
diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
index ccde59c..3ad450c 100644
--- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
+++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.LockComponent;
import org.apache.hadoop.hive.metastore.api.LockLevel;
import org.apache.hadoop.hive.metastore.api.LockRequest;
@@ -57,7 +56,6 @@ import org.apache.iceberg.Snapshot;
import org.apache.iceberg.SnapshotSummary;
import org.apache.iceberg.TableMetadata;
import org.apache.iceberg.TableProperties;
-import org.apache.iceberg.common.DynMethods;
import org.apache.iceberg.exceptions.AlreadyExistsException;
import org.apache.iceberg.exceptions.CommitFailedException;
import org.apache.iceberg.exceptions.CommitStateUnknownException;
@@ -93,14 +91,7 @@ public class HiveTableOperations extends BaseMetastoreTableOperations {
private static final long HIVE_LOCK_CHECK_MIN_WAIT_MS_DEFAULT = 50; // 50 milliseconds
private static final long HIVE_LOCK_CHECK_MAX_WAIT_MS_DEFAULT = 5 * 1000; // 5 seconds
private static final long HIVE_TABLE_LEVEL_LOCK_EVICT_MS_DEFAULT = TimeUnit.MINUTES.toMillis(10);
- private static final DynMethods.UnboundMethod ALTER_TABLE = DynMethods.builder("alter_table")
- .impl(IMetaStoreClient.class, "alter_table_with_environmentContext",
- String.class, String.class, Table.class, EnvironmentContext.class)
- .impl(IMetaStoreClient.class, "alter_table",
- String.class, String.class, Table.class, EnvironmentContext.class)
- .impl(IMetaStoreClient.class, "alter_table",
- String.class, String.class, Table.class)
- .build();
+
private static final BiMap<String, String> ICEBERG_TO_HMS_TRANSLATION = ImmutableBiMap.of(
// gc.enabled in Iceberg and external.table.purge in Hive are meant to do the same things but with different names
GC_ENABLED, "external.table.purge"
@@ -310,10 +301,7 @@ public class HiveTableOperations extends BaseMetastoreTableOperations {
void persistTable(Table hmsTable, boolean updateHiveTable) throws TException, InterruptedException {
if (updateHiveTable) {
metaClients.run(client -> {
- EnvironmentContext envContext = new EnvironmentContext(
- ImmutableMap.of(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE)
- );
- ALTER_TABLE.invoke(client, database, tableName, hmsTable, envContext);
+ MetastoreUtil.alterTable(client, database, tableName, hmsTable);
return null;
});
} else {
diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java
index ad0ec80..76363f1 100644
--- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java
+++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java
@@ -19,12 +19,28 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.iceberg.common.DynMethods;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+
public class MetastoreUtil {
// this class is unique to Hive3 and cannot be found in Hive2, therefore a good proxy to see if
// we are working against Hive3 dependencies
private static final String HIVE3_UNIQUE_CLASS = "org.apache.hadoop.hive.serde2.io.DateWritableV2";
+ private static final DynMethods.UnboundMethod ALTER_TABLE = DynMethods.builder("alter_table")
+ .impl(IMetaStoreClient.class, "alter_table_with_environmentContext",
+ String.class, String.class, Table.class, EnvironmentContext.class)
+ .impl(IMetaStoreClient.class, "alter_table",
+ String.class, String.class, Table.class, EnvironmentContext.class)
+ .impl(IMetaStoreClient.class, "alter_table",
+ String.class, String.class, Table.class)
+ .build();
+
private static final boolean HIVE3_PRESENT_ON_CLASSPATH = detectHive3();
private MetastoreUtil() {
@@ -37,6 +53,17 @@ public class MetastoreUtil {
return HIVE3_PRESENT_ON_CLASSPATH;
}
+ /**
+ * Calls alter_table method using the metastore client. If possible, an environmental context will be used that
+ * turns off stats updates to avoid recursive listing.
+ */
+ public static void alterTable(IMetaStoreClient client, String databaseName, String tblName, Table table) {
+ EnvironmentContext envContext = new EnvironmentContext(
+ ImmutableMap.of(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE)
+ );
+ ALTER_TABLE.invoke(client, databaseName, tblName, table, envContext);
+ }
+
private static boolean detectHive3() {
try {
Class.forName(HIVE3_UNIQUE_CLASS);