You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2026/04/17 23:38:39 UTC
(impala) branch master updated: IMPALA-14896: Do less GetPartialCatalogObject for Iceberg tables
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new c101e5f58 IMPALA-14896: Do less GetPartialCatalogObject for Iceberg tables
c101e5f58 is described below
commit c101e5f5832ed3082670ab293bc265564ef48d7a
Author: Csaba Ringhofer <cs...@cloudera.com>
AuthorDate: Thu Apr 16 17:18:02 2026 +0200
IMPALA-14896: Do less GetPartialCatalogObject for Iceberg tables
Before this patch 5 GetPartialCatalogObject RPCs were needed
to catalog to load an Iceberg table. This patch removes 2
(loadPartitionsByRefs, loadPartitionList) which have trivial results
for Iceberg tables and don't need caching. This logic was already
implemented for Iceberg REST catalog in IcebergMetaProvider.
Change-Id: I9bbb6cf08a15e8ad2a76acd86380b87f06fc25a5
Reviewed-on: http://gerrit.cloudera.org:8080/24197
Reviewed-by: Noemi Pap-Takacs <np...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../impala/catalog/local/CatalogdMetaProvider.java | 15 +++++++++++
.../impala/catalog/local/DirectMetaProvider.java | 6 +++++
.../impala/catalog/local/IcebergMetaProvider.java | 30 ++++++++++++++--------
.../apache/impala/catalog/local/MetaProvider.java | 1 +
4 files changed, 42 insertions(+), 10 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java b/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
index c48191c25..7115641c5 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
@@ -70,6 +70,7 @@ import org.apache.impala.catalog.HdfsCachePool;
import org.apache.impala.catalog.HdfsPartitionLocationCompressor;
import org.apache.impala.catalog.HdfsStorageDescriptor;
import org.apache.impala.catalog.HdfsTable;
+import org.apache.impala.catalog.IcebergTable;
import org.apache.impala.catalog.ImpaladCatalog.ObjectUpdateSequencer;
import org.apache.impala.catalog.Principal;
import org.apache.impala.catalog.PrincipalPrivilege;
@@ -982,6 +983,10 @@ public class CatalogdMetaProvider implements MetaProvider {
@Override
public List<PartitionRef> loadPartitionList(final TableMetaRef table)
throws TException {
+ // For Iceberg tables, delegate to the dummy Iceberg partition loading logic
+ if (table.isIceberg()) {
+ return IcebergMetaProvider.loadPartitionListHelper();
+ }
PartitionListCacheKey key = new PartitionListCacheKey((TableMetaRefImpl) table);
return (List<PartitionRef>) loadWithCaching("partition list for " + table,
PARTITION_LIST_STATS_CATEGORY, key, new Callable<List<PartitionRef>>() {
@@ -1019,6 +1024,11 @@ public class CatalogdMetaProvider implements MetaProvider {
throws CatalogException, TException {
Preconditions.checkArgument(table instanceof TableMetaRefImpl);
TableMetaRefImpl refImpl = (TableMetaRefImpl)table;
+ // For Iceberg tables, delegate to the dummy Iceberg partition loading logic
+ if (table.isIceberg()) {
+ return IcebergMetaProvider.loadPartitionsByRefsHelper(refImpl.msTable_);
+ }
+
Stopwatch sw = Stopwatch.createStarted();
// Load what we can from the cache.
Map<PartitionRef, PartitionMetadata> refToMeta = loadPartitionsFromCache(refImpl,
@@ -2126,6 +2136,11 @@ public class CatalogdMetaProvider implements MetaProvider {
@Override
public long getLoadedTimeMs() { return loadedTimeMs_; }
+
+ @Override
+ public boolean isIceberg() {
+ return IcebergTable.isIcebergTable(msTable_);
+ }
}
/**
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/DirectMetaProvider.java b/fe/src/main/java/org/apache/impala/catalog/local/DirectMetaProvider.java
index 626879f11..58deeecd1 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/DirectMetaProvider.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/DirectMetaProvider.java
@@ -43,6 +43,7 @@ import org.apache.impala.catalog.FileMetadataLoader;
import org.apache.impala.catalog.Function;
import org.apache.impala.catalog.HdfsCachePool;
import org.apache.impala.catalog.HdfsPartitionLocationCompressor;
+import org.apache.impala.catalog.IcebergTable;
import org.apache.impala.catalog.HdfsStorageDescriptor;
import org.apache.impala.catalog.MetaStoreClientPool;
import org.apache.impala.catalog.MetaStoreClientPool.MetaStoreClient;
@@ -559,6 +560,11 @@ class DirectMetaProvider implements MetaProvider {
public long getLoadedTimeMs() {
return 0;
}
+
+ @Override
+ public boolean isIceberg() {
+ return IcebergTable.isIcebergTable(msTable_);
+ }
}
@Override
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/IcebergMetaProvider.java b/fe/src/main/java/org/apache/impala/catalog/local/IcebergMetaProvider.java
index 053a4b765..d21a088f5 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/IcebergMetaProvider.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/IcebergMetaProvider.java
@@ -249,6 +249,14 @@ public class IcebergMetaProvider implements MetaProvider {
throws MetaException, TException {
TableMetaRefImpl ref = (TableMetaRefImpl)table;
Preconditions.checkState(!ref.isPartitioned());
+ return loadPartitionListHelper();
+ }
+
+ /**
+ * Helper method to load partition list for Iceberg tables.
+ * Iceberg tables are always handled as unpartitioned in catalog.
+ */
+ public static List<PartitionRef> loadPartitionListHelper() {
return ImmutableList.of(new PartitionRefImpl(PartitionRefImpl.UNPARTITIONED_NAME));
}
@@ -263,21 +271,18 @@ public class IcebergMetaProvider implements MetaProvider {
TableMetaRef table, List<String> partitionColumnNames,
ListMap<TNetworkAddress> hostIndex,
List<PartitionRef> partitionRefs) throws CatalogException, TException {
- Map<String, PartitionMetadata> ret = new HashMap<>();
- ret.put("", new PartitionMetadataImpl(((TableMetaRefImpl)table).msTable_));
- return ret;
+ TableMetaRefImpl ref = (TableMetaRefImpl)table;
+ return loadPartitionsByRefsHelper(ref.msTable_);
}
/**
- * We model partitions slightly differently to Hive. So, in the case of an
- * unpartitioned table, we have to create a fake Partition object which has the
- * metadata of the table.
+ * Helper method to load dummy partition for Iceberg tables.
+ * Iceberg tables are always handled as unpartitioned in catalog.
*/
- private Map<String, PartitionMetadata> loadUnpartitionedPartition(
- TableMetaRefImpl table, List<PartitionRef> partitionRefs,
- ListMap<TNetworkAddress> hostIndex) throws CatalogException {
+ public static Map<String, PartitionMetadata> loadPartitionsByRefsHelper(
+ Table msTable) throws CatalogException {
Map<String, PartitionMetadata> ret = new HashMap<>();
- ret.put("", new PartitionMetadataImpl(((TableMetaRefImpl)table).msTable_));
+ ret.put("", new PartitionMetadataImpl(msTable));
return ret;
}
@@ -524,6 +529,11 @@ public class IcebergMetaProvider implements MetaProvider {
public long getLoadedTimeMs() {
return loadingTimeMs_;
}
+
+ @Override
+ public boolean isIceberg() {
+ return true;
+ }
}
@Override
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java b/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java
index 0666d07d9..a97929742 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java
@@ -178,6 +178,7 @@ public interface MetaProvider {
List<VirtualColumn> getVirtualColumns();
long getCatalogVersion();
long getLoadedTimeMs();
+ boolean isIceberg();
}
/**