You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by bo...@apache.org on 2022/03/04 15:25:10 UTC
[impala] branch master updated: IMPALA-10737: Optimize the number of Iceberg API Metadata requests
This is an automated email from the ASF dual-hosted git repository.
boroknagyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new cd10acd IMPALA-10737: Optimize the number of Iceberg API Metadata requests
cd10acd is described below
commit cd10acdbb15c74eae21fa630bcb174104283560a
Author: Tamas Mate <tm...@cloudera.com>
AuthorDate: Fri Feb 11 15:40:53 2022 +0100
IMPALA-10737: Optimize the number of Iceberg API Metadata requests
Iceberg stores the table metadata next to the data files, when this is
accessed through the Iceberg API a filesystem call is executed (HDFS,
S3, ADLS). These calls were used in various places during query
processing and this patch unifies the Iceberg metadata request in the
CatalogD similar to other metadata requests:
- CatalogD loads and caches the org.apache.iceberg.BaseTable object.
- ImpalaDs requests the org.apache.iceberg.BaseTable from the
CatalogD and caches it as well.
As a result REFRESH/INVALIDATE METADATA is required to reload any
Iceberg metadata changes and the metadata load time is improved.
This improvement is more significant for smaller queries, where the
metadata request has larger impact on the query execution time.
Testing:
- Passed Iceberg E2E tests.
Change-Id: I9e62a1fb9753ea1b022c7763047d9ccfd1d27d62
Reviewed-on: http://gerrit.cloudera.org:8080/18226
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Zoltan Borok-Nagy <bo...@cloudera.com>
---
common/thrift/CatalogObjects.thrift | 15 ++--
common/thrift/CatalogService.thrift | 14 +---
.../impala/catalog/CatalogServiceCatalog.java | 1 -
.../org/apache/impala/catalog/FeIcebergTable.java | 88 +++++++++++++++++-----
.../org/apache/impala/catalog/IcebergTable.java | 37 +++++----
.../catalog/IcebergTableLoadingException.java | 32 ++++++++
.../impala/catalog/iceberg/IcebergCtasTarget.java | 12 +++
.../impala/catalog/local/CatalogdMetaProvider.java | 66 +++++++++-------
.../impala/catalog/local/DirectMetaProvider.java | 13 +---
.../impala/catalog/local/LocalIcebergTable.java | 64 ++++++++--------
.../apache/impala/catalog/local/LocalTable.java | 2 +-
.../apache/impala/catalog/local/MetaProvider.java | 8 +-
.../apache/impala/service/CatalogOpExecutor.java | 10 ++-
.../java/org/apache/impala/service/Frontend.java | 2 +-
.../java/org/apache/impala/util/IcebergUtil.java | 32 +-------
.../impala/catalog/local/LocalCatalogTest.java | 8 +-
.../queries/QueryTest/iceberg-insert.test | 1 +
17 files changed, 230 insertions(+), 175 deletions(-)
diff --git a/common/thrift/CatalogObjects.thrift b/common/thrift/CatalogObjects.thrift
index 1cc729e..ef3ad40 100644
--- a/common/thrift/CatalogObjects.thrift
+++ b/common/thrift/CatalogObjects.thrift
@@ -574,18 +574,19 @@ struct TIcebergTable {
2: required list<TIcebergPartitionSpec> partition_spec
3: required i32 default_partition_spec_id
// Map from 128-bit Murmur3 hash of data file path to its file descriptor
- 4: optional map<string,THdfsFileDesc> path_hash_to_file_descriptor
- // Iceberg snapshot id of the table
- 5: optional i64 snapshot_id
+ 4: optional map<string, THdfsFileDesc> path_hash_to_file_descriptor
// Iceberg 'write.parquet.compression-codec' and 'write.parquet.compression-level' table
// properties
- 6: optional TCompressionCodec parquet_compression_codec
+ 5: optional TCompressionCodec parquet_compression_codec
// Iceberg 'write.parquet.row-group-size-bytes' table property
- 7: optional i64 parquet_row_group_size
+ 6: optional i64 parquet_row_group_size
// Iceberg 'write.parquet.page-size-bytes' and 'write.parquet.dict-size-bytes' table
// properties
- 8: optional i64 parquet_plain_page_size;
- 9: optional i64 parquet_dict_page_size;
+ 7: optional i64 parquet_plain_page_size;
+ 8: optional i64 parquet_dict_page_size;
+ // Serialized org.apache.iceberg.BaseTable object, it is used to execute operations
+ // on the encapsulated Iceberg org.apache.iceberg.TableMetadata.
+ 9: optional binary iceberg_base_table;
}
// Represents a table or view.
diff --git a/common/thrift/CatalogService.thrift b/common/thrift/CatalogService.thrift
index 886c41b..ba98df9 100644
--- a/common/thrift/CatalogService.thrift
+++ b/common/thrift/CatalogService.thrift
@@ -390,9 +390,8 @@ struct TTableInfoSelector {
// it in cases the clients do need HMS partition structs.
12: bool want_hms_partition
- // The response should contain information about the Iceberg snapshot, i.e. the snapshot
- // id and the file descriptors.
- 13: bool want_iceberg_snapshot
+ // The response should contain information about the Iceberg table.
+ 13: bool want_iceberg_table
}
// Returned information about a particular partition.
@@ -442,11 +441,6 @@ struct TPartialPartitionInfo {
13: optional CatalogObjects.THdfsPartitionLocation location
}
-struct TIcebergSnapshot {
- 1: required i64 snapshot_id
- 2: optional map<string, CatalogObjects.THdfsFileDesc> iceberg_file_desc_map
-}
-
// Returned information about a Table, as selected by TTableInfoSelector.
struct TPartialTableInfo {
1: optional hive_metastore.Table hms_table
@@ -487,8 +481,8 @@ struct TPartialTableInfo {
// the description of how a prefix is computed.
11: optional list<string> partition_prefixes
- // Iceberg snapshot information
- 12: optional TIcebergSnapshot iceberg_snapshot
+ // Iceberg table information
+ 12: optional CatalogObjects.TIcebergTable iceberg_table
}
struct TBriefTableMeta {
diff --git a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
index c85af60..c3dc4e7 100644
--- a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
@@ -59,7 +59,6 @@ import org.apache.impala.authorization.AuthorizationPolicy;
import org.apache.impala.catalog.FeFsTable.Utils;
import org.apache.impala.catalog.HdfsPartition.FileDescriptor;
import org.apache.impala.catalog.MetaStoreClientPool.MetaStoreClient;
-import org.apache.impala.catalog.events.EventFactory;
import org.apache.impala.catalog.events.ExternalEventsProcessor;
import org.apache.impala.catalog.events.MetastoreEvents.EventFactoryForSyncToLatestEvent;
import org.apache.impala.catalog.events.MetastoreEvents.MetastoreEventFactory;
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java b/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java
index e52e91f..6a020d3 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java
@@ -17,7 +17,12 @@
package org.apache.impala.catalog;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -31,6 +36,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.iceberg.BaseTable;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.PartitionField;
import org.apache.iceberg.PartitionSpec;
@@ -51,7 +57,6 @@ import org.apache.impala.thrift.THdfsTable;
import org.apache.impala.thrift.THdfsPartition;
import org.apache.impala.thrift.TIcebergCatalog;
import org.apache.impala.thrift.TIcebergFileFormat;
-import org.apache.impala.thrift.TIcebergSnapshot;
import org.apache.impala.thrift.TIcebergTable;
import org.apache.impala.thrift.TNetworkAddress;
import org.apache.impala.thrift.TResultSet;
@@ -87,6 +92,18 @@ public interface FeIcebergTable extends FeFsTable {
TIcebergCatalog getIcebergCatalog();
/**
+ * Returns the BaseTable that stores the metadata loaded by Iceberg.
+ */
+ BaseTable getIcebergBaseTable();
+
+ /**
+ * Returns the TableMetadata from the BaseTable.
+ */
+ default TableMetadata getIcebergTableMetadata() {
+ return getIcebergBaseTable().operations().current();
+ }
+
+ /**
* Return Iceberg catalog location, we use this location to load metadata from Iceberg
* When using 'hadoop.tables', this value equals to table location
* When using 'hadoop.catalog', this value equals to 'iceberg.catalog_location'
@@ -266,18 +283,11 @@ public interface FeIcebergTable extends FeFsTable {
/**
* Current snapshot id of the table.
*/
- long snapshotId();
-
- /**
- * Utility class to hold information about Iceberg snapshots.
- */
- public static class Snapshot {
- public Snapshot(long snapshotId, Map<String, FileDescriptor> pathHashToFileDescMap) {
- this.snapshotId = snapshotId;
- this.pathHashToFileDescMap = pathHashToFileDescMap;
+ default long snapshotId() {
+ if (getIcebergBaseTable().currentSnapshot() != null) {
+ return getIcebergTableMetadata().currentSnapshot().snapshotId();
}
- public long snapshotId;
- public Map<String, FileDescriptor> pathHashToFileDescMap;
+ return -1;
}
/**
@@ -305,7 +315,7 @@ public interface FeIcebergTable extends FeFsTable {
resultSchema.addToColumns(new TColumn("Field Partition Transform",
Type.STRING.toThrift()));
- TableMetadata metadata = IcebergUtil.getIcebergTableMetadata(table);
+ TableMetadata metadata = table.getIcebergTableMetadata();
if (!metadata.specs().isEmpty()) {
// Just show the current PartitionSpec from Iceberg table metadata
PartitionSpec latestSpec = metadata.spec();
@@ -416,7 +426,6 @@ public interface FeIcebergTable extends FeFsTable {
tIcebergTable.setPath_hash_to_file_descriptor(
convertPathHashToFileDescMap(icebergTable));
- tIcebergTable.setSnapshot_id(icebergTable.snapshotId());
tIcebergTable.setParquet_compression_codec(
icebergTable.getIcebergParquetCompressionCodec());
tIcebergTable.setParquet_row_group_size(
@@ -424,7 +433,9 @@ public interface FeIcebergTable extends FeFsTable {
tIcebergTable.setParquet_plain_page_size(
icebergTable.getIcebergParquetPlainPageSize());
tIcebergTable.setParquet_dict_page_size(
- icebergTable.getIcebergParquetDictPageSize());
+ icebergTable.getIcebergParquetDictPageSize());
+ tIcebergTable.setIceberg_base_table(serializeIcebergBaseTable(
+ icebergTable.getIcebergBaseTable()));
return tIcebergTable;
}
@@ -462,11 +473,48 @@ public interface FeIcebergTable extends FeFsTable {
return fileDescMap;
}
- public static TIcebergSnapshot createTIcebergSnapshot(FeIcebergTable icebergTable) {
- TIcebergSnapshot snapshot = new TIcebergSnapshot();
- snapshot.setSnapshot_id(icebergTable.snapshotId());
- snapshot.setIceberg_file_desc_map(convertPathHashToFileDescMap(icebergTable));
- return snapshot;
+ public static byte[] serializeIcebergBaseTable(BaseTable baseTable) {
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ ObjectOutputStream objectOutputStream = null;
+ byte[] result = new byte[]{};
+ try {
+ objectOutputStream = new ObjectOutputStream(byteArrayOutputStream);
+ objectOutputStream.writeObject(baseTable);
+ objectOutputStream.flush();
+ result = byteArrayOutputStream.toByteArray();
+ } catch (IOException e) {
+ LOG.warn("Failed to serialize Iceberg BaseTable: " + e.getMessage());
+ }
+ finally {
+ try {
+ byteArrayOutputStream.close();
+ } catch (IOException ex) {
+ // ignore close exception
+ }
+ }
+ return result;
+ }
+
+ public static BaseTable deserializeIcebergBaseTable(byte[] serializedBaseTable)
+ throws TableLoadingException {
+ Object baseTable = null;
+ ByteArrayInputStream inputStream = new ByteArrayInputStream(serializedBaseTable);
+ ObjectInput objectInput = null;
+ try {
+ objectInput = new ObjectInputStream(inputStream);
+ baseTable = objectInput.readObject();
+ } catch (Exception e) {
+ throw new TableLoadingException("Failed to deserialize Iceberg BaseTable.", e);
+ } finally {
+ try {
+ if (objectInput != null) {
+ objectInput.close();
+ }
+ } catch (IOException ex) {
+ // ignore close exception
+ }
+ }
+ return (BaseTable)baseTable;
}
/**
diff --git a/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java b/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
index 42a6ae9..519bbb2 100644
--- a/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
@@ -28,8 +28,8 @@ import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.iceberg.Schema;
+import org.apache.iceberg.BaseTable;
import org.apache.iceberg.TableMetadata;
-import org.apache.iceberg.types.Types;
import org.apache.impala.analysis.IcebergPartitionField;
import org.apache.impala.analysis.IcebergPartitionSpec;
import org.apache.impala.analysis.IcebergPartitionTransform;
@@ -39,7 +39,6 @@ import org.apache.impala.thrift.TCompressionCodec;
import org.apache.impala.thrift.TGetPartialCatalogObjectRequest;
import org.apache.impala.thrift.TGetPartialCatalogObjectResponse;
import org.apache.impala.thrift.THdfsCompression;
-import org.apache.impala.thrift.THdfsFileDesc;
import org.apache.impala.thrift.THdfsTable;
import org.apache.impala.thrift.TIcebergCatalog;
import org.apache.impala.thrift.TIcebergFileFormat;
@@ -165,7 +164,8 @@ public class IcebergTable extends Table implements FeIcebergTable {
// Treat iceberg table as a non-partitioned hdfs table in backend
private HdfsTable hdfsTable_;
- private long snapshotId_ = -1;
+ // Treat iceberg table as a non-partitioned hdfs table in backend
+ private BaseTable icebergBaseTable_;
protected IcebergTable(org.apache.hadoop.hive.metastore.api.Table msTable,
Db db, String name, String owner) {
@@ -204,6 +204,11 @@ public class IcebergTable extends Table implements FeIcebergTable {
}
@Override
+ public BaseTable getIcebergBaseTable() {
+ return icebergBaseTable_;
+ }
+
+ @Override
public TCatalogObjectType getCatalogObjectType() {
return TCatalogObjectType.TABLE;
}
@@ -299,11 +304,6 @@ public class IcebergTable extends Table implements FeIcebergTable {
}
@Override
- public long snapshotId() {
- return snapshotId_;
- }
-
- @Override
public TTable toThrift() {
TTable table = super.toThrift();
table.setTable_type(TTableType.ICEBERG_TABLE);
@@ -333,10 +333,8 @@ public class IcebergTable extends Table implements FeIcebergTable {
final Timer.Context ctxStorageLdTime =
getMetrics().getTimer(Table.LOAD_DURATION_STORAGE_METADATA).time();
try {
- TableMetadata metadata = IcebergUtil.getIcebergTableMetadata(this);
- if (metadata.currentSnapshot() != null) {
- snapshotId_ = metadata.currentSnapshot().snapshotId();
- }
+ icebergBaseTable_ = (BaseTable)IcebergUtil.loadTable(this);
+ TableMetadata metadata = getIcebergTableMetadata();
loadSchemaFromIceberg(metadata);
// Loading hdfs table after loaded schema from Iceberg,
// in case we create external Iceberg table skipping column info in sql.
@@ -350,8 +348,8 @@ public class IcebergTable extends Table implements FeIcebergTable {
pathHashToFileDescMap_ = Utils.loadAllPartition(this);
loadAllColumnStats(msClient);
} catch (Exception e) {
- throw new TableLoadingException("Error loading metadata for Iceberg table " +
- icebergTableLocation_, e);
+ throw new IcebergTableLoadingException("Error loading metadata for Iceberg table "
+ + icebergTableLocation_, e);
} finally {
storageMetadataLoadTime_ = ctxStorageLdTime.stop();
}
@@ -422,10 +420,10 @@ public class IcebergTable extends Table implements FeIcebergTable {
// so there's no need for translation.
pathHashToFileDescMap_ = FeIcebergTable.Utils.loadFileDescMapFromThrift(
ticeberg.getPath_hash_to_file_descriptor(), null, null);
- snapshotId_ = ticeberg.getSnapshot_id();
hdfsTable_.loadFromThrift(thriftTable);
- TableMetadata metadata = IcebergUtil.getIcebergTableMetadata(this);
- icebergSchema_ = metadata.schema();
+ icebergBaseTable_ = FeIcebergTable.Utils.deserializeIcebergBaseTable(
+ ticeberg.getIceberg_base_table());
+ icebergSchema_ = icebergBaseTable_.operations().current().schema();
}
private List<IcebergPartitionSpec> loadPartitionBySpecsFromThrift(
@@ -482,9 +480,8 @@ public class IcebergTable extends Table implements FeIcebergTable {
Map<HdfsPartition, TPartialPartitionInfo> missingPartialInfos = new HashMap<>();
TGetPartialCatalogObjectResponse resp =
getHdfsTable().getPartialInfo(req, missingPartialInfos);
- if (req.table_info_selector.want_iceberg_snapshot) {
- resp.table_info.setIceberg_snapshot(
- FeIcebergTable.Utils.createTIcebergSnapshot(this));
+ if (req.table_info_selector.want_iceberg_table) {
+ resp.table_info.setIceberg_table(Utils.getTIcebergTable(this));
if (!resp.table_info.isSetNetwork_addresses()) {
resp.table_info.setNetwork_addresses(getHostIndex().getList());
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/IcebergTableLoadingException.java b/fe/src/main/java/org/apache/impala/catalog/IcebergTableLoadingException.java
new file mode 100644
index 0000000..4b2d691
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/catalog/IcebergTableLoadingException.java
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.catalog;
+
+/**
+ * Thrown when the iceberg table metadata cannot be loaded due to an error.
+ */
+public class IcebergTableLoadingException extends TableLoadingException {
+
+ public IcebergTableLoadingException(String s, Throwable cause) {
+ super(s, cause);
+ }
+
+ public IcebergTableLoadingException(String s) {
+ super(s);
+ }
+};
\ No newline at end of file
diff --git a/fe/src/main/java/org/apache/impala/catalog/iceberg/IcebergCtasTarget.java b/fe/src/main/java/org/apache/impala/catalog/iceberg/IcebergCtasTarget.java
index e0c986f..3657801 100644
--- a/fe/src/main/java/org/apache/impala/catalog/iceberg/IcebergCtasTarget.java
+++ b/fe/src/main/java/org/apache/impala/catalog/iceberg/IcebergCtasTarget.java
@@ -26,9 +26,11 @@ import java.util.Set;
import com.google.common.base.Preconditions;
+import org.apache.iceberg.BaseTable;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
+import org.apache.iceberg.TableMetadata;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.types.TypeUtil;
@@ -254,6 +256,16 @@ public class IcebergCtasTarget extends CtasTargetTable implements FeIcebergTable
return -1;
}
+ @Override
+ public BaseTable getIcebergBaseTable() {
+ return null;
+ }
+
+ @Override
+ public TableMetadata getIcebergTableMetadata() {
+ return null;
+ }
+
public void addColumn(IcebergColumn col) {
colsByPos_.add(col);
colsByName_.put(col.getName().toLowerCase(), col);
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java b/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
index 7dff758..d60bb64 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
@@ -54,7 +54,6 @@ import org.apache.impala.catalog.Catalog;
import org.apache.impala.catalog.CatalogDeltaLog;
import org.apache.impala.catalog.CatalogException;
import org.apache.impala.catalog.CatalogObjectCache;
-import org.apache.impala.catalog.FeIcebergTable;
import org.apache.impala.catalog.Function;
import org.apache.impala.catalog.HdfsCachePool;
import org.apache.impala.catalog.HdfsPartition.FileDescriptor;
@@ -84,7 +83,6 @@ import org.apache.impala.thrift.TFunctionName;
import org.apache.impala.thrift.TGetPartialCatalogObjectRequest;
import org.apache.impala.thrift.TGetPartialCatalogObjectResponse;
import org.apache.impala.thrift.THdfsFileDesc;
-import org.apache.impala.thrift.TIcebergSnapshot;
import org.apache.impala.thrift.TNetworkAddress;
import org.apache.impala.thrift.TPartialPartitionInfo;
import org.apache.impala.thrift.TPartialTableInfo;
@@ -106,7 +104,6 @@ import org.ehcache.sizeof.SizeOf;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.base.Stopwatch;
@@ -609,7 +606,7 @@ public class CatalogdMetaProvider implements MetaProvider {
req.catalog_info_selector.want_db_names = true;
TGetPartialCatalogObjectResponse resp = sendRequest(req);
checkResponse(resp.catalog_info != null && resp.catalog_info.db_names != null,
- req, "missing table names");
+ req, "missing database names");
return ImmutableList.copyOf(resp.catalog_info.db_names);
}
});
@@ -1024,34 +1021,35 @@ public class CatalogdMetaProvider implements MetaProvider {
return ret;
}
- /**
- * Utility function for to retrieve table info with Iceberg snapshot. Exists for testing
- * purposes, use loadIcebergSnapshot() which translates the file descriptors to the
- * table's host index.
- */
- @VisibleForTesting
- TPartialTableInfo loadTableInfoWithIcebergSnapshot(final TableMetaRef table)
- throws TException {
+ @Override
+ public TPartialTableInfo loadIcebergTable(final TableMetaRef table) throws TException {
Preconditions.checkArgument(table instanceof TableMetaRefImpl);
- TGetPartialCatalogObjectRequest req = newReqForTable(table);
- req.table_info_selector.want_iceberg_snapshot = true;
- TGetPartialCatalogObjectResponse resp = sendRequest(req);
- return resp.table_info;
+ return loadTableInfoWithIcebergTable(table);
}
- @Override
- public FeIcebergTable.Snapshot loadIcebergSnapshot(final TableMetaRef table,
- ListMap<TNetworkAddress> hostIndex)
+ /**
+ * Utility function for to retrieve table info with Iceberg table metadata.
+ */
+ TPartialTableInfo loadTableInfoWithIcebergTable(final TableMetaRef table)
throws TException {
- TPartialTableInfo tableInfo = loadTableInfoWithIcebergSnapshot(table);
- Map<String, FileDescriptor> pathToFds =
- FeIcebergTable.Utils.loadFileDescMapFromThrift(
- tableInfo.getIceberg_snapshot().getIceberg_file_desc_map(),
- tableInfo.getNetwork_addresses(),
- hostIndex);
- return new FeIcebergTable.Snapshot(
- tableInfo.getIceberg_snapshot().getSnapshot_id(),
- pathToFds);
+ Preconditions.checkArgument(table instanceof TableMetaRefImpl);
+ TableMetaRefImpl tableRef = (TableMetaRefImpl)table;
+ String itemStr = "iceberg metadata for " + tableRef.dbName_ + "."
+ + tableRef.tableName_;
+ IcebergMetaCacheKey cacheKey = new IcebergMetaCacheKey(tableRef);
+ return loadWithCaching(itemStr, TABLE_METADATA_CACHE_CATEGORY, cacheKey,
+ new Callable<TPartialTableInfo>() {
+ @Override
+ public TPartialTableInfo call() throws Exception {
+ TGetPartialCatalogObjectRequest req = newReqForTable(table);
+ req.table_info_selector.want_iceberg_table = true;
+ TGetPartialCatalogObjectResponse resp = sendRequest(req);
+ checkResponse(resp.table_info != null &&
+ resp.table_info.iceberg_table != null, req,
+ "missing Iceberg table metadata");
+ return resp.getTable_info();
+ }
+ });
}
private ImmutableList<FileDescriptor> convertThriftFdList(List<THdfsFileDesc> thriftFds,
@@ -1942,6 +1940,18 @@ public class CatalogdMetaProvider implements MetaProvider {
}
}
+ /**
+ * Cache key for an entry storing Iceberg table metadata.
+ *
+ * Values for these keys are 'TPartialTableInfo' objects.
+ */
+ private static class IcebergMetaCacheKey extends VersionedTableCacheKey {
+
+ public IcebergMetaCacheKey(TableMetaRefImpl table) {
+ super(table);
+ }
+ }
+
@VisibleForTesting
static class SizeOfWeigher implements Weigher<Object, Object> {
// Bypass flyweight objects like small boxed integers, Boolean.TRUE, enums, etc.
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/DirectMetaProvider.java b/fe/src/main/java/org/apache/impala/catalog/local/DirectMetaProvider.java
index 41c7d15..36c33ef 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/DirectMetaProvider.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/DirectMetaProvider.java
@@ -19,18 +19,13 @@ package org.apache.impala.catalog.local;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
import com.google.common.base.Stopwatch;
-import com.google.common.collect.Iterables;
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
@@ -44,7 +39,6 @@ import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.TableMeta;
import org.apache.impala.authorization.AuthorizationPolicy;
import org.apache.impala.catalog.CatalogException;
-import org.apache.impala.catalog.FeIcebergTable;
import org.apache.impala.catalog.FileMetadataLoader;
import org.apache.impala.catalog.Function;
import org.apache.impala.catalog.HdfsCachePool;
@@ -60,8 +54,8 @@ import org.apache.impala.compat.MetastoreShim;
import org.apache.impala.service.BackendConfig;
import org.apache.impala.thrift.TBackendGflags;
import org.apache.impala.thrift.TBriefTableMeta;
-import org.apache.impala.thrift.TIcebergSnapshot;
import org.apache.impala.thrift.TNetworkAddress;
+import org.apache.impala.thrift.TPartialTableInfo;
import org.apache.impala.thrift.TValidWriteIdList;
import org.apache.impala.util.AcidUtils;
import org.apache.impala.util.ListMap;
@@ -546,10 +540,9 @@ class DirectMetaProvider implements MetaProvider {
}
@Override
- public FeIcebergTable.Snapshot loadIcebergSnapshot(final TableMetaRef table,
- ListMap<TNetworkAddress> hostIndex)
+ public TPartialTableInfo loadIcebergTable(final TableMetaRef table)
throws TException {
throw new NotImplementedException(
- "loadIcebergSnapshot() is not implemented for DirectMetaProvider");
+ "loadIcebergTable() is not implemented for DirectMetaProvider");
}
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalIcebergTable.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalIcebergTable.java
index 09bec8f..077e9d9 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalIcebergTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalIcebergTable.java
@@ -17,7 +17,6 @@
package org.apache.impala.catalog.local;
-import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
@@ -26,8 +25,8 @@ import java.util.Set;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.iceberg.BaseTable;
import org.apache.iceberg.Schema;
-import org.apache.iceberg.TableMetadata;
import org.apache.impala.analysis.IcebergPartitionSpec;
import org.apache.impala.catalog.CatalogObject;
import org.apache.impala.catalog.Column;
@@ -42,13 +41,12 @@ import org.apache.impala.thrift.THdfsPartition;
import org.apache.impala.thrift.THdfsTable;
import org.apache.impala.thrift.TIcebergCatalog;
import org.apache.impala.thrift.TIcebergFileFormat;
-import org.apache.impala.thrift.TIcebergSnapshot;
-import org.apache.impala.thrift.TNetworkAddress;
+import org.apache.impala.thrift.TIcebergTable;
+import org.apache.impala.thrift.TPartialTableInfo;
import org.apache.impala.thrift.TTableDescriptor;
import org.apache.impala.thrift.TTableType;
import org.apache.impala.util.IcebergSchemaConverter;
import org.apache.impala.util.IcebergUtil;
-import org.apache.thrift.TException;
import com.google.common.base.Preconditions;
import com.google.errorprone.annotations.Immutable;
@@ -70,29 +68,32 @@ public class LocalIcebergTable extends LocalTable implements FeIcebergTable {
private int defaultPartitionSpecId_;
private Map<String, FileDescriptor> pathHashToFileDescMap_;
private LocalFsTable localFsTable_;
- private long snapshotId_ = -1;
private Schema icebergSchema_;
- static LocalTable loadFromIceberg(LocalDb db, Table msTable,
+ // BaseTable contains the metadata that Iceberg keeps track of.
+ private BaseTable icebergBaseTable_;
+
+ /**
+ * Loads the Iceberg metadata from the CatalogD then initializes a LocalIcebergTable.
+ */
+ static LocalTable loadIcebergTableViaMetaProvider(LocalDb db, Table msTable,
MetaProvider.TableMetaRef ref) throws TableLoadingException {
Preconditions.checkNotNull(db);
Preconditions.checkNotNull(msTable);
try {
- TableParams params = new TableParams(msTable);
- TableMetadata metadata =
- IcebergUtil.getIcebergTableMetadata(params.icebergCatalog_,
- IcebergUtil.getIcebergTableIdentifier(msTable),
- params.icebergCatalogLocation_,
- msTable.getParameters());
- List<Column> iceColumns =
- IcebergSchemaConverter.convertToImpalaSchema(metadata.schema());
+ TPartialTableInfo tableInfo =
+ db.getCatalog().getMetaProvider().loadIcebergTable(ref);
+ TIcebergTable table = tableInfo.getIceberg_table();
+ BaseTable icebergBaseTable = FeIcebergTable.Utils.deserializeIcebergBaseTable(
+ table.getIceberg_base_table());
+ List<Column> iceColumns = IcebergSchemaConverter.convertToImpalaSchema(
+ icebergBaseTable.operations().current().schema());
validateColumns(iceColumns, msTable.getSd().getCols());
ColumnMap colMap = new ColumnMap(iceColumns,
/*numClusteringCols=*/ 0,
db.getName() + "." + msTable.getTableName(),
/*isFullAcidSchema=*/false);
-
- return new LocalIcebergTable(db, msTable, ref, colMap, metadata);
+ return new LocalIcebergTable(db, msTable, ref, colMap, tableInfo, icebergBaseTable);
} catch (Exception e) {
String fullTableName = msTable.getDbName() + "." + msTable.getTableName();
throw new TableLoadingException(
@@ -101,25 +102,20 @@ public class LocalIcebergTable extends LocalTable implements FeIcebergTable {
}
private LocalIcebergTable(LocalDb db, Table msTable, MetaProvider.TableMetaRef ref,
- ColumnMap cmap, TableMetadata metadata)
+ ColumnMap cmap, TPartialTableInfo tableInfo, BaseTable icebergBaseTable)
throws TableLoadingException {
super(db, msTable, ref, cmap);
+ Preconditions.checkNotNull(tableInfo);
localFsTable_ = LocalFsTable.load(db, msTable, ref);
tableParams_ = new TableParams(msTable);
- FeIcebergTable.Snapshot snapshot;
- try {
- snapshot = db_.getCatalog().getMetaProvider().loadIcebergSnapshot(
- ref, getHostIndex());
- Preconditions.checkNotNull(snapshot);
- snapshotId_ = snapshot.snapshotId;
- pathHashToFileDescMap_ = snapshot.pathHashToFileDescMap;
- } catch (TException e) {
- throw new TableLoadingException(String.format(
- "Failed to load table: %s.%s", msTable.getDbName(), msTable.getTableName()), e);
- }
- partitionSpecs_ = Utils.loadPartitionSpecByIceberg(metadata);
- defaultPartitionSpecId_ = metadata.defaultSpecId();
- icebergSchema_ = metadata.schema();
+ pathHashToFileDescMap_ = FeIcebergTable.Utils.loadFileDescMapFromThrift(
+ tableInfo.getIceberg_table().getPath_hash_to_file_descriptor(),
+ tableInfo.getNetwork_addresses(),
+ getHostIndex());
+ icebergBaseTable_ = icebergBaseTable;
+ partitionSpecs_ = Utils.loadPartitionSpecByIceberg(getIcebergTableMetadata());
+ defaultPartitionSpecId_ = getIcebergTableMetadata().defaultSpecId();
+ icebergSchema_ = getIcebergTableMetadata().schema();
icebergFileFormat_ = IcebergUtil.getIcebergFileFormat(msTable);
icebergParquetCompressionCodec_ = Utils.getIcebergParquetCompressionCodec(msTable);
icebergParquetRowGroupSize_ = Utils.getIcebergParquetRowGroupSize(msTable);
@@ -204,8 +200,8 @@ public class LocalIcebergTable extends LocalTable implements FeIcebergTable {
}
@Override
- public long snapshotId() {
- return snapshotId_;
+ public BaseTable getIcebergBaseTable() {
+ return icebergBaseTable_;
}
@Override
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
index f09b656..f056c2b 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
@@ -104,7 +104,7 @@ abstract class LocalTable implements FeTable {
} else if (KuduTable.isKuduTable(msTbl)) {
t = LocalKuduTable.loadFromKudu(db, msTbl, ref);
} else if (IcebergTable.isIcebergTable(msTbl)) {
- t = LocalIcebergTable.loadFromIceberg(db, msTbl, ref);
+ t = LocalIcebergTable.loadIcebergTableViaMetaProvider(db, msTbl, ref);
} else if (DataSourceTable.isDataSourceTable(msTbl)) {
// TODO(todd) support datasource table
} else if (HdfsFileFormat.isHdfsInputFormatClass(
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java b/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java
index 524871d..ca437be 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.impala.authorization.AuthorizationPolicy;
import org.apache.impala.catalog.CatalogException;
-import org.apache.impala.catalog.FeIcebergTable;
import org.apache.impala.catalog.Function;
import org.apache.impala.catalog.HdfsCachePool;
import org.apache.impala.catalog.HdfsPartition.FileDescriptor;
@@ -37,8 +36,8 @@ import org.apache.impala.catalog.HdfsStorageDescriptor;
import org.apache.impala.catalog.SqlConstraints;
import org.apache.impala.common.Pair;
import org.apache.impala.thrift.TBriefTableMeta;
-import org.apache.impala.thrift.TIcebergSnapshot;
import org.apache.impala.thrift.TNetworkAddress;
+import org.apache.impala.thrift.TPartialTableInfo;
import org.apache.impala.thrift.TValidWriteIdList;
import org.apache.impala.util.ListMap;
import org.apache.thrift.TException;
@@ -126,10 +125,9 @@ public interface MetaProvider {
List<String> colNames) throws TException;
/**
- * Loads Iceberg snapshot information, i.e. snapshot id and file descriptors.
+ * Loads Iceberg related table metadata.
*/
- public FeIcebergTable.Snapshot loadIcebergSnapshot(final TableMetaRef table,
- ListMap<TNetworkAddress> hostIndex)
+ public TPartialTableInfo loadIcebergTable(final TableMetaRef table)
throws TException;
/**
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index 4d6a59a..e3b24db 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -102,6 +102,7 @@ import org.apache.impala.catalog.Db;
import org.apache.impala.catalog.FeCatalogUtils;
import org.apache.impala.catalog.FileMetadataLoadOpts;
import org.apache.impala.catalog.IcebergTable;
+import org.apache.impala.catalog.IcebergTableLoadingException;
import org.apache.impala.catalog.FeFsPartition;
import org.apache.impala.catalog.FeFsTable;
import org.apache.impala.catalog.FeIcebergTable;
@@ -6061,8 +6062,13 @@ public class CatalogOpExecutor {
// ACID tables, there is a Jira to cover this: HIVE-22062.
// 2: If no need for a full table reload then fetch partition level
// writeIds and reload only the ones that changed.
- updatedThriftTable = catalog_.reloadTable(tbl, req, resultType, cmdString,
- -1);
+ try {
+ updatedThriftTable = catalog_.reloadTable(tbl, req, resultType, cmdString,
+ -1);
+ } catch (IcebergTableLoadingException e) {
+ updatedThriftTable = catalog_.invalidateTable(
+ req.getTable_name(), tblWasRemoved, dbWasAdded);
+ }
}
} else {
// Table was loaded from scratch, so it's already "refreshed".
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index a921bb9..865da57 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -1117,7 +1117,7 @@ public class Frontend {
FeTable feTable = getCatalog().getTable(dbName, tableName);
Preconditions.checkState(feTable instanceof FeIcebergTable);
FeIcebergTable feIcebergTable = (FeIcebergTable) feTable;
- TableMetadata metadata = IcebergUtil.getIcebergTableMetadata(feIcebergTable);
+ TableMetadata metadata = feIcebergTable.getIcebergTableMetadata();
Table table = IcebergUtil.loadTable(feIcebergTable);
Set<Long> ancestorIds = Sets.newHashSet(IcebergUtil.currentAncestorIds(table));
diff --git a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
index feefd1f..02d33e0 100644
--- a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
@@ -40,9 +40,7 @@ import com.google.common.hash.Hashing;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
-import org.apache.impala.catalog.IcebergStructField;
import org.apache.impala.common.Pair;
-import org.apache.iceberg.BaseTable;
import org.apache.iceberg.Transaction;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.DataFile;
@@ -55,10 +53,8 @@ import org.apache.iceberg.Schema;
import org.apache.iceberg.Snapshot;
import org.apache.iceberg.StructLike;
import org.apache.iceberg.Table;
-import org.apache.iceberg.TableMetadata;
import org.apache.iceberg.mr.Catalogs;
import org.apache.iceberg.transforms.PartitionSpecVisitor;
-import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.types.Conversions;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
@@ -79,14 +75,12 @@ import org.apache.impala.catalog.iceberg.IcebergCatalog;
import org.apache.impala.catalog.iceberg.IcebergCatalogs;
import org.apache.impala.common.ImpalaRuntimeException;
import org.apache.impala.thrift.TCompressionCodec;
-import org.apache.impala.thrift.TCreateTableParams;
import org.apache.impala.thrift.THdfsCompression;
import org.apache.impala.thrift.THdfsFileFormat;
import org.apache.impala.thrift.TIcebergCatalog;
import org.apache.impala.thrift.TIcebergFileFormat;
import org.apache.impala.thrift.TIcebergPartitionField;
import org.apache.impala.thrift.TIcebergPartitionSpec;
-import org.apache.impala.thrift.TIcebergPartitionTransform;
import org.apache.impala.thrift.TIcebergPartitionTransformType;
public class IcebergUtil {
@@ -143,27 +137,6 @@ public class IcebergUtil {
}
/**
- * Get TableMetadata by FeIcebergTable
- */
- public static TableMetadata getIcebergTableMetadata(FeIcebergTable table)
- throws TableLoadingException {
- BaseTable iceTable = (BaseTable)IcebergUtil.loadTable(table);
- return iceTable.operations().current();
- }
-
- /**
- * Get TableMetadata by related info tableName is table full name, usually
- * database.table
- */
- public static TableMetadata getIcebergTableMetadata(TIcebergCatalog catalog,
- TableIdentifier tableId, String location, Map<String, String> tableProps)
- throws TableLoadingException {
- BaseTable baseTable = (BaseTable)IcebergUtil.loadTable(catalog,
- tableId, location, tableProps);
- return baseTable.operations().current();
- }
-
- /**
* Get Iceberg table identifier by table property
*/
public static TableIdentifier getIcebergTableIdentifier(FeIcebergTable table) {
@@ -194,7 +167,7 @@ public class IcebergUtil {
*/
public static Transaction getIcebergTransaction(FeIcebergTable feTable)
throws TableLoadingException, ImpalaRuntimeException {
- return getIcebergCatalog(feTable).loadTable(feTable).newTransaction();
+ return feTable.getIcebergBaseTable().newTransaction();
}
/**
@@ -557,8 +530,7 @@ public class IcebergUtil {
private static TableScan createScanAsOf(FeIcebergTable table,
TimeTravelSpec timeTravelSpec) throws TableLoadingException {
- BaseTable baseTable = (BaseTable)IcebergUtil.loadTable(table);
- TableScan scan = baseTable.newScan();
+ TableScan scan = table.getIcebergBaseTable().newScan();
if (timeTravelSpec == null) {
scan = scan.useSnapshot(table.snapshotId());
} else {
diff --git a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
index 61721e4..c72de6c 100644
--- a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
@@ -26,7 +26,6 @@ import java.util.Set;
import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
import org.apache.hive.service.rpc.thrift.TGetColumnsReq;
-import org.apache.hive.service.rpc.thrift.TGetSchemasReq;
import org.apache.hive.service.rpc.thrift.TGetTablesReq;
import org.apache.impala.analysis.Expr;
import org.apache.impala.analysis.ToSqlUtils;
@@ -46,14 +45,12 @@ import org.apache.impala.catalog.Type;
import org.apache.impala.service.BackendConfig;
import org.apache.impala.service.FeSupport;
import org.apache.impala.service.Frontend;
-import org.apache.impala.service.MetadataOp;
import org.apache.impala.testutil.TestUtils;
import org.apache.impala.thrift.TCatalogObjectType;
import org.apache.impala.thrift.TMetadataOpRequest;
import org.apache.impala.thrift.TMetadataOpcode;
import org.apache.impala.thrift.TNetworkAddress;
import org.apache.impala.thrift.TPartialTableInfo;
-import org.apache.impala.thrift.TResultRow;
import org.apache.impala.thrift.TResultSet;
import org.apache.impala.util.ListMap;
import org.apache.impala.util.MetaStoreUtil;
@@ -61,7 +58,6 @@ import org.apache.impala.util.PatternMatcher;
import org.hamcrest.CoreMatchers;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import com.google.common.base.Joiner;
@@ -264,12 +260,12 @@ public class LocalCatalogTest {
LocalIcebergTable t = (LocalIcebergTable)catalog_.getTable(
"functional_parquet", "iceberg_partitioned");
Map<String, FileDescriptor> localTblFdMap = t.getPathHashToFileDescMap();
- TPartialTableInfo tblInfo = provider_.loadTableInfoWithIcebergSnapshot(t.ref_);
+ TPartialTableInfo tblInfo = provider_.loadTableInfoWithIcebergTable(t.ref_);
ListMap<TNetworkAddress> catalogdHostIndexes = new ListMap<>();
catalogdHostIndexes.populate(tblInfo.getNetwork_addresses());
Map<String, FileDescriptor> catalogFdMap =
FeIcebergTable.Utils.loadFileDescMapFromThrift(
- tblInfo.getIceberg_snapshot().getIceberg_file_desc_map(),
+ tblInfo.getIceberg_table().getPath_hash_to_file_descriptor(),
null, null);
for (Map.Entry<String, FileDescriptor> entry : localTblFdMap.entrySet()) {
String path = entry.getKey();
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-insert.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-insert.test
index a05b440..bffbd97 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-insert.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-insert.test
@@ -239,6 +239,7 @@ drop table iceberg_hive_cat
====
---- QUERY
# The data has been purged, so querying the external table fails.
+refresh iceberg_hive_cat_ext_2;
select * from iceberg_hive_cat_ext_2
---- CATCH
Table does not exist