You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by pv...@apache.org on 2026/07/24 09:31:00 UTC
(iceberg) branch main updated: Hive: Use server-side filter to list Iceberg tables in HiveCatalog (#17317)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 8f4843394c Hive: Use server-side filter to list Iceberg tables in HiveCatalog (#17317)
8f4843394c is described below
commit 8f4843394cbfca3583108b8f6174f86f8580071b
Author: terrytlu <te...@tencent.com>
AuthorDate: Fri Jul 24 17:30:51 2026 +0800
Hive: Use server-side filter to list Iceberg tables in HiveCatalog (#17317)
---
.../java/org/apache/iceberg/hive/HiveCatalog.java | 40 +++++++++++-
.../org/apache/iceberg/hive/TestHiveCatalog.java | 71 ++++++++++++++++++++++
2 files changed, 108 insertions(+), 3 deletions(-)
diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
index 7df5ad96be..287b35082b 100644
--- a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
+++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
@@ -20,6 +20,7 @@ package org.apache.iceberg.hive;
import java.io.IOException;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@@ -34,6 +35,7 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.iceberg.BaseMetastoreTableOperations;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.CatalogUtil;
@@ -167,18 +169,19 @@ public class HiveCatalog extends BaseMetastoreViewCatalog
String database = namespace.level(0);
try {
- List<String> tableNames = clients.run(client -> client.getAllTables(database));
List<TableIdentifier> tableIdentifiers;
if (listAllTables) {
+ List<String> tableNames = clients.run(client -> client.getAllTables(database));
tableIdentifiers =
tableNames.stream()
.map(t -> TableIdentifier.of(namespace, t))
.collect(Collectors.toList());
} else {
+ // Retrieving only the matching table names from HMS via server-side filter
tableIdentifiers =
- listIcebergTables(
- tableNames, namespace, BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE);
+ listIcebergTablesByFilter(
+ namespace, BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE);
}
LOG.debug(
@@ -354,6 +357,37 @@ public class HiveCatalog extends BaseMetastoreViewCatalog
.collect(Collectors.toList());
}
+ /**
+ * Lists the {@link TableIdentifier} of all Iceberg tables under the given database.
+ *
+ * @param namespace the namespace corresponding to the database
+ * @param tableTypeValue the value of {@code parameters.table_type} to filter on.
+ */
+ private List<TableIdentifier> listIcebergTablesByFilter(
+ Namespace namespace, String tableTypeValue) throws TException, InterruptedException {
+ String database = namespace.level(0);
+ // HMS normalizes the `table_type` parameter value to upper case when persisting it
+ // (e.g. "iceberg" is stored as "ICEBERG"), so the filter value must be upper-cased to match.
+ // Note: `like` (rather than `=`) is used intentionally. Some HMS backends (e.g. Derby and
+ // Oracle, where PARAM_VALUE is a CLOB) do not support `=` comparison on property values and
+ // fail the filter query (see HIVE-21614); newer HMS versions internally rewrite `=` to `like`
+ // for this reason. As the value carries no `%`/`_` wildcards, `like` is equivalent to `=` here.
+ String filter =
+ hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS
+ + BaseMetastoreTableOperations.TABLE_TYPE_PROP
+ + " like \""
+ + tableTypeValue.toUpperCase(Locale.ROOT)
+ + "\"";
+
+ List<String> icebergTableNames =
+ clients.run(
+ client -> client.listTableNamesByFilter(database, filter, (short) -1 /* no limit */));
+
+ return icebergTableNames.stream()
+ .map(tableName -> TableIdentifier.of(namespace, tableName))
+ .collect(Collectors.toList());
+ }
+
@SuppressWarnings("checkstyle:CyclomaticComplexity")
private void renameTableOrView(
TableIdentifier from,
diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java
index 0cf2882a79..bc5e66d0f4 100644
--- a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java
+++ b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java
@@ -45,8 +45,11 @@ import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.iceberg.BaseMetastoreTableOperations;
import org.apache.iceberg.CachingCatalog;
@@ -78,6 +81,7 @@ import org.apache.iceberg.exceptions.NoSuchNamespaceException;
import org.apache.iceberg.exceptions.NoSuchTableException;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.transforms.Transforms;
@@ -1266,4 +1270,71 @@ public class TestHiveCatalog extends CatalogTests<HiveCatalog> {
assertThat(base64EncodedHash).isNull();
}
}
+
+ @Test
+ public void testListTablesFiltersOutNonIcebergTables() throws TException {
+ Namespace ns = Namespace.of(DB_NAME);
+ TableIdentifier icebergTable = TableIdentifier.of(ns, "iceberg_tbl");
+ TableIdentifier hiveTable = TableIdentifier.of(ns, "hive_tbl");
+
+ catalog.createTable(icebergTable, getTestSchema());
+ HIVE_METASTORE_EXTENSION
+ .metastoreClient()
+ .createTable(createNonIcebergTable(hiveTable.name(), TableType.EXTERNAL_TABLE));
+
+ try {
+ assertThat(catalog.listTables(ns)).containsExactly(icebergTable);
+ } finally {
+ catalog.dropTable(icebergTable);
+ HIVE_METASTORE_EXTENSION.metastoreClient().dropTable(DB_NAME, hiveTable.name());
+ }
+ }
+
+ @Test
+ public void testListTablesByFilterReturnsAllIcebergTables() throws Exception {
+ Namespace ns = Namespace.of(DB_NAME);
+ TableIdentifier t1 = TableIdentifier.of(ns, "t1");
+ TableIdentifier t2 = TableIdentifier.of(ns, "t2");
+
+ catalog.createTable(t1, getTestSchema());
+ catalog.createTable(t2, getTestSchema());
+
+ try {
+ assertThat(catalog.listTables(ns)).containsExactlyInAnyOrder(t1, t2);
+ } finally {
+ catalog.dropTable(t1);
+ catalog.dropTable(t2);
+ }
+ }
+
+ private org.apache.hadoop.hive.metastore.api.Table createNonIcebergTable(
+ String hiveTableName, TableType type) {
+ StorageDescriptor sd =
+ new StorageDescriptor(
+ Lists.newArrayList(),
+ temp.resolve(hiveTableName).toString(),
+ "org.apache.hadoop.mapred.TextInputFormat",
+ "org.apache.hadoop.mapred.TextOutputFormat",
+ false,
+ -1,
+ new SerDeInfo(
+ null, "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", Maps.newHashMap()),
+ Lists.newArrayList(),
+ Lists.newArrayList(),
+ Maps.newHashMap());
+
+ return new org.apache.hadoop.hive.metastore.api.Table(
+ hiveTableName,
+ DB_NAME,
+ "test_owner",
+ 0,
+ 0,
+ 0,
+ sd,
+ Lists.newArrayList(),
+ Maps.newHashMap(),
+ null,
+ null,
+ type.name());
+ }
}