You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by wz...@apache.org on 2024/01/11 18:07:57 UTC
(impala) 01/05: IMPALA-12495: Describe statement for Iceberg metadata tables
This is an automated email from the ASF dual-hosted git repository.
wzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 9cc0aa886cbbaac3f69c7ea91b397ebbea739986
Author: Tamas Mate <tm...@cloudera.com>
AuthorDate: Wed Nov 8 13:07:32 2023 +0100
IMPALA-12495: Describe statement for Iceberg metadata tables
Iceberg metadata tables are virtual tables and their schemata are
predefined by the Iceberg library. This commit extends the DESCRIBE
<table> statement, so the users can print the table description of these
tables.
Metadata tables do not exist in the HMS, therefore
DESCRIBE FORMATTED|EXTENDED statements are not permitted.
Testing:
- Added E2E tests
Change-Id: Ibe22f271a59a6885035991c09b5101193ade6e97
Reviewed-on: http://gerrit.cloudera.org:8080/20695
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/service/frontend.cc | 3 +
common/thrift/Frontend.thrift | 7 +-
.../java/org/apache/impala/analysis/Analyzer.java | 22 +-
.../apache/impala/analysis/DescribeTableStmt.java | 14 ++
.../org/apache/impala/analysis/FromClause.java | 3 -
.../java/org/apache/impala/analysis/TableName.java | 3 +-
.../impala/service/DescribeResultFactory.java | 18 +-
.../java/org/apache/impala/service/Frontend.java | 70 ++++--
.../org/apache/impala/service/JniFrontend.java | 11 +-
.../authorization/AuthorizationTestBase.java | 8 +-
.../queries/QueryTest/iceberg-metadata-tables.test | 268 ++++++++++++++++++++-
11 files changed, 377 insertions(+), 50 deletions(-)
diff --git a/be/src/service/frontend.cc b/be/src/service/frontend.cc
index e8f53f3d7..0a9ce18dc 100644
--- a/be/src/service/frontend.cc
+++ b/be/src/service/frontend.cc
@@ -186,6 +186,9 @@ Status Frontend::DescribeTable(const TDescribeTableParams& params,
tparams.__set_output_style(params.output_style);
if (params.__isset.table_name) tparams.__set_table_name(params.table_name);
if (params.__isset.result_struct) tparams.__set_result_struct(params.result_struct);
+ if (params.__isset.metadata_table_name) {
+ tparams.__set_metadata_table_name(params.metadata_table_name);
+ }
tparams.__set_session(session);
return JniUtil::CallJniMethod(fe_, describe_table_id_, tparams, response);
}
diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift
index e7118ed19..f19f532c2 100644
--- a/common/thrift/Frontend.thrift
+++ b/common/thrift/Frontend.thrift
@@ -187,11 +187,14 @@ struct TDescribeTableParams {
// Set when describing a table.
2: optional CatalogObjects.TTableName table_name
+ // Set for metadata tables
+ 3: optional string metadata_table_name
+
// Set when describing a path to a nested collection.
- 3: optional Types.TColumnType result_struct
+ 4: optional Types.TColumnType result_struct
// Session state for the user who initiated this request.
- 4: optional Query.TSessionState session
+ 5: optional Query.TSessionState session
}
// Results of a call to describeDb() and describeTable()
diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
index 91e3a0151..f6ce7b2bd 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
@@ -3563,6 +3563,9 @@ public class Analyzer {
*/
public FeTable getTable(TableName tblName, boolean mustExist)
throws AnalysisException, TableLoadingException {
+ if (IcebergMetadataTable.isIcebergMetadataTable(tblName.toPath())) {
+ return getMetadataVirtualTable(tblName.toPath());
+ }
FeTable table = globalState_.stmtTableCache.tables.get(tblName);
if (table == null) {
if (!mustExist) {
@@ -3603,13 +3606,15 @@ public class Analyzer {
}
/**
- * Adds a new Iceberg metadata table to the stmt table cache. At this point it is
- * unknown if the base table is loaded for scanning as well, therefore the original
- * table is kept. The metadata table will have its vTbl field filled, while the original
- * table gets a new key without the vTbl field.
- * 'tblRefPath' parameter has to be an IcebergMetadataTable reference path.
+ * Retrieves the Iceberg metadata table from the stmtTableCache if the Iceberg metadata
+ * table exists or creates and adds it to the stmtTableCache if it does not exist. At
+ * this point it is unknown if the base table is loaded for scanning as well, therefore
+ * the original table is kept. The metadata table will have its vTbl field filled, while
+ * the original table gets a new key without the vTbl field. 'tblRefPath' parameter has
+ * to be an IcebergMetadataTable reference path. Returns the the Iceberg metadata table.
*/
- public void addMetadataVirtualTable(List<String> tblRefPath) throws AnalysisException {
+ public FeTable getMetadataVirtualTable(List<String> tblRefPath)
+ throws AnalysisException {
Preconditions.checkArgument(IcebergMetadataTable.isIcebergMetadataTable(tblRefPath));
try {
TableName catalogTableName = new TableName(tblRefPath.get(0),
@@ -3619,11 +3624,14 @@ public class Analyzer {
// The catalog table (the base of the virtual table) has been loaded and cached
// under the name of the virtual table.
FeTable catalogTable = getStmtTableCache().tables.get(virtualTableName);
- if (catalogTable instanceof IcebergMetadataTable || catalogTable == null) return;
+ if (catalogTable instanceof IcebergMetadataTable || catalogTable == null) {
+ return catalogTable;
+ }
IcebergMetadataTable virtualTable =
new IcebergMetadataTable(catalogTable, tblRefPath.get(2));
getStmtTableCache().tables.put(catalogTableName, catalogTable);
getStmtTableCache().tables.put(virtualTableName, virtualTable);
+ return virtualTable;
} catch (ImpalaRuntimeException e) {
throw new AnalysisException("Could not create metadata table for table "
+ "reference: " + StringUtils.join(tblRefPath, "."), e);
diff --git a/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java b/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java
index ec99f6f62..4830354df 100644
--- a/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java
@@ -25,6 +25,7 @@ import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.FeTable;
import org.apache.impala.catalog.StructType;
import org.apache.impala.catalog.TableLoadingException;
+import org.apache.impala.catalog.iceberg.IcebergMetadataTable;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.thrift.TDescribeOutputStyle;
import org.apache.impala.thrift.TDescribeTableParams;
@@ -124,6 +125,7 @@ public class DescribeTableStmt extends StatementBase {
// all columns returning an empty result due to insufficient VIEW_METADATA privilege.
analyzer.getTable(table_.getTableName(), /* add column-level privilege */ true,
Privilege.ANY);
+ checkMinimalForIcebergMetadataTable();
// Describing a table.
if (path_.destTable() != null) return;
@@ -150,6 +152,14 @@ public class DescribeTableStmt extends StatementBase {
}
}
+ private void checkMinimalForIcebergMetadataTable() throws AnalysisException {
+ if (table_ instanceof IcebergMetadataTable &&
+ outputStyle_ != TDescribeOutputStyle.MINIMAL) {
+ throw new AnalysisException("DESCRIBE FORMATTED|EXTENDED cannot refer to a "
+ + "metadata table.");
+ }
+ }
+
public TDescribeTableParams toThrift() {
TDescribeTableParams params = new TDescribeTableParams();
params.setOutput_style(outputStyle_);
@@ -158,6 +168,10 @@ public class DescribeTableStmt extends StatementBase {
} else {
Preconditions.checkNotNull(table_);
params.setTable_name(table_.getTableName().toThrift());
+ if (table_ instanceof IcebergMetadataTable) {
+ params.setMetadata_table_name(((IcebergMetadataTable)table_).
+ getMetadataTableName());
+ }
}
return params;
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/FromClause.java b/fe/src/main/java/org/apache/impala/analysis/FromClause.java
index 656c609c4..306b5af86 100644
--- a/fe/src/main/java/org/apache/impala/analysis/FromClause.java
+++ b/fe/src/main/java/org/apache/impala/analysis/FromClause.java
@@ -84,9 +84,6 @@ public class FromClause extends StmtNode implements Iterable<TableRef> {
boolean hasJoiningUnnest = false;
for (int i = 0; i < tableRefs_.size(); ++i) {
TableRef tblRef = tableRefs_.get(i);
- if (IcebergMetadataTable.isIcebergMetadataTable(tblRef.getPath())) {
- analyzer.addMetadataVirtualTable(tblRef.getPath());
- }
tblRef = analyzer.resolveTableRef(tblRef);
tableRefs_.set(i, Preconditions.checkNotNull(tblRef));
tblRef.setLeftTblRef(leftTblRef);
diff --git a/fe/src/main/java/org/apache/impala/analysis/TableName.java b/fe/src/main/java/org/apache/impala/analysis/TableName.java
index d7cdcc6c6..1f0efc67e 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TableName.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TableName.java
@@ -140,9 +140,10 @@ public class TableName {
}
public List<String> toPath() {
- List<String> result = Lists.newArrayListWithCapacity(2);
+ List<String> result = Lists.newArrayListWithCapacity(3);
if (db_ != null) result.add(db_);
result.add(tbl_);
+ if (vTbl_ != null && !vTbl_.isEmpty()) result.add(vTbl_);
return result;
}
diff --git a/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java b/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java
index 888df7b65..6ce58e5c1 100644
--- a/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java
+++ b/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java
@@ -33,6 +33,7 @@ import org.apache.impala.catalog.IcebergColumn;
import org.apache.impala.catalog.KuduColumn;
import org.apache.impala.catalog.StructField;
import org.apache.impala.catalog.StructType;
+import org.apache.impala.catalog.iceberg.IcebergMetadataTable;
import org.apache.impala.common.ImpalaRuntimeException;
import org.apache.impala.compat.MetastoreShim;
import org.apache.impala.thrift.TColumnValue;
@@ -351,4 +352,19 @@ public class DescribeResultFactory {
}
return descResult;
}
-}
+
+ /**
+ * Builds a TDescribeResult for an Iceberg metadata table from an IcebergTable and a
+ * metadata table name.
+ *
+ * This describe request is issued against a VirtualTable which only exists in the
+ * Analyzer's StmtTableCache. Therefore, to get the columns of an IcebergMetadataTable
+ * it is simpler to re-create this object than to extract those from a new
+ * org.apache.iceberg.Table object or to send it over.
+ */
+ public static TDescribeResult buildIcebergMetadataDescribeMinimalResult(FeTable table,
+ String vTableName) throws ImpalaRuntimeException {
+ IcebergMetadataTable metadataTable = new IcebergMetadataTable(table, vTableName);
+ return buildIcebergDescribeMinimalResult(metadataTable.getColumns());
+ }
+}
\ No newline at end of file
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index 31f9a570d..d030655fd 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -127,9 +127,11 @@ import org.apache.impala.catalog.ImpaladTableUsageTracker;
import org.apache.impala.catalog.MaterializedViewHdfsTable;
import org.apache.impala.catalog.MetaStoreClientPool;
import org.apache.impala.catalog.MetaStoreClientPool.MetaStoreClient;
+import org.apache.impala.catalog.StructType;
import org.apache.impala.catalog.TableLoadingException;
import org.apache.impala.catalog.Type;
import org.apache.impala.catalog.local.InconsistentMetadataFetchException;
+import org.apache.impala.catalog.iceberg.IcebergMetadataTable;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.FileSystemUtil;
import org.apache.impala.common.ImpalaException;
@@ -169,6 +171,7 @@ import org.apache.impala.thrift.TDescribeHistoryParams;
import org.apache.impala.thrift.TDescribeOutputStyle;
import org.apache.impala.thrift.TDescribeResult;
import org.apache.impala.thrift.TImpalaTableType;
+import org.apache.impala.thrift.TDescribeTableParams;
import org.apache.impala.thrift.TIcebergDmlFinalizeParams;
import org.apache.impala.thrift.TIcebergOperation;
import org.apache.impala.thrift.TExecRequest;
@@ -664,7 +667,8 @@ public class Frontend {
columns.add(new TColumn("encoding", Type.STRING.toThrift()));
columns.add(new TColumn("compression", Type.STRING.toThrift()));
columns.add(new TColumn("block_size", Type.STRING.toThrift()));
- } else if (descStmt.getTable() instanceof FeIcebergTable
+ } else if ((descStmt.getTable() instanceof FeIcebergTable
+ || descStmt.getTable() instanceof IcebergMetadataTable)
&& descStmt.getOutputStyle() == TDescribeOutputStyle.MINIMAL) {
columns.add(new TColumn("nullable", Type.STRING.toThrift()));
}
@@ -1656,24 +1660,34 @@ public class Frontend {
* Throws an exception if the table or db is not found or if there is an error loading
* the table metadata.
*/
- public TDescribeResult describeTable(TTableName tableName,
- TDescribeOutputStyle outputStyle, User user) throws ImpalaException {
- RetryTracker retries = new RetryTracker(
- String.format("fetching table %s.%s", tableName.db_name, tableName.table_name));
- while (true) {
- try {
- return doDescribeTable(tableName, outputStyle, user);
- } catch(InconsistentMetadataFetchException e) {
- retries.handleRetryOrThrow(e);
+ public TDescribeResult describeTable(TDescribeTableParams params, User user)
+ throws ImpalaException {
+ if (params.isSetTable_name()) {
+ RetryTracker retries = new RetryTracker(
+ String.format("fetching table %s.%s", params.table_name.db_name,
+ params.table_name.table_name));
+ while (true) {
+ try {
+ return doDescribeTable(params.table_name, params.output_style, user,
+ params.metadata_table_name);
+ } catch(InconsistentMetadataFetchException e) {
+ retries.handleRetryOrThrow(e);
+ }
}
+ } else {
+ Preconditions.checkState(params.output_style == TDescribeOutputStyle.MINIMAL);
+ Preconditions.checkNotNull(params.result_struct);
+ StructType structType = (StructType)Type.fromThrift(params.result_struct);
+ return DescribeResultFactory.buildDescribeMinimalResult(structType);
}
}
- private TDescribeResult doDescribeTable(TTableName tableName,
- TDescribeOutputStyle outputStyle, User user) throws ImpalaException {
- FeTable table = getCatalog().getTable(tableName.db_name,
- tableName.table_name);
- List<Column> filteredColumns;
+ /**
+ * Filters out columns that the user is not authorized to see.
+ */
+ private List<Column> filterAuthorizedColumnsForDescribeTable(FeTable table, User user)
+ throws InternalException {
+ List<Column> authFilteredColumns;
if (authzFactory_.getAuthorizationConfig().isEnabled()) {
// First run a table check
PrivilegeRequest privilegeRequest = new PrivilegeRequestBuilder(
@@ -1681,7 +1695,7 @@ public class Frontend {
.allOf(Privilege.VIEW_METADATA).onTable(table).build();
if (!authzChecker_.get().hasAccess(user, privilegeRequest)) {
// Filter out columns that the user is not authorized to see.
- filteredColumns = new ArrayList<Column>();
+ authFilteredColumns = new ArrayList<Column>();
for (Column col: table.getColumnsInHiveOrder()) {
String colName = col.getName();
privilegeRequest = new PrivilegeRequestBuilder(
@@ -1690,22 +1704,37 @@ public class Frontend {
.onColumn(table.getDb().getName(),
table.getName(), colName, table.getOwnerUser()).build();
if (authzChecker_.get().hasAccess(user, privilegeRequest)) {
- filteredColumns.add(col);
+ authFilteredColumns.add(col);
}
}
} else {
// User has table-level access
- filteredColumns = table.getColumnsInHiveOrder();
+ authFilteredColumns = table.getColumnsInHiveOrder();
}
} else {
// Authorization is disabled
- filteredColumns = table.getColumnsInHiveOrder();
+ authFilteredColumns = table.getColumnsInHiveOrder();
}
+ return authFilteredColumns;
+ }
+
+ private TDescribeResult doDescribeTable(TTableName tableName,
+ TDescribeOutputStyle outputStyle, User user, String vTableName)
+ throws ImpalaException {
+ FeTable table = getCatalog().getTable(tableName.db_name,
+ tableName.table_name);
+ List<Column> filteredColumns = filterAuthorizedColumnsForDescribeTable(table, user);
if (outputStyle == TDescribeOutputStyle.MINIMAL) {
if (table instanceof FeKuduTable) {
return DescribeResultFactory.buildKuduDescribeMinimalResult(filteredColumns);
} else if (table instanceof FeIcebergTable) {
- return DescribeResultFactory.buildIcebergDescribeMinimalResult(filteredColumns);
+ if (vTableName == null) {
+ return DescribeResultFactory.buildIcebergDescribeMinimalResult(filteredColumns);
+ } else {
+ Preconditions.checkArgument(vTableName != null);
+ return DescribeResultFactory.buildIcebergMetadataDescribeMinimalResult(table,
+ vTableName);
+ }
} else {
return DescribeResultFactory.buildDescribeMinimalResult(
Column.columnsToStruct(filteredColumns));
@@ -1713,6 +1742,7 @@ public class Frontend {
} else {
Preconditions.checkArgument(outputStyle == TDescribeOutputStyle.FORMATTED ||
outputStyle == TDescribeOutputStyle.EXTENDED);
+ Preconditions.checkArgument(vTableName == null);
TDescribeResult result = DescribeResultFactory.buildDescribeFormattedResult(table,
filteredColumns);
// Filter out LOCATION text
diff --git a/fe/src/main/java/org/apache/impala/service/JniFrontend.java b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
index 1f2213e15..4c6d05f7c 100644
--- a/fe/src/main/java/org/apache/impala/service/JniFrontend.java
+++ b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
@@ -495,18 +495,9 @@ public class JniFrontend {
Preconditions.checkNotNull(frontend_);
TDescribeTableParams params = new TDescribeTableParams();
JniUtil.deserializeThrift(protocolFactory_, params, thriftDescribeTableParams);
-
Preconditions.checkState(params.isSetTable_name() ^ params.isSetResult_struct());
User user = new User(TSessionStateUtil.getEffectiveUser(params.getSession()));
- TDescribeResult result = null;
- if (params.isSetTable_name()) {
- result = frontend_.describeTable(params.getTable_name(), params.output_style, user);
- } else {
- Preconditions.checkState(params.output_style == TDescribeOutputStyle.MINIMAL);
- StructType structType = (StructType)Type.fromThrift(params.result_struct);
- result = DescribeResultFactory.buildDescribeMinimalResult(structType);
- }
-
+ TDescribeResult result = frontend_.describeTable(params, user);
try {
TSerializer serializer = new TSerializer(protocolFactory_);
return serializer.serialize(result);
diff --git a/fe/src/test/java/org/apache/impala/authorization/AuthorizationTestBase.java b/fe/src/test/java/org/apache/impala/authorization/AuthorizationTestBase.java
index b9b34c6c0..3e551227a 100644
--- a/fe/src/test/java/org/apache/impala/authorization/AuthorizationTestBase.java
+++ b/fe/src/test/java/org/apache/impala/authorization/AuthorizationTestBase.java
@@ -38,6 +38,7 @@ import org.apache.impala.testutil.ImpaladTestCatalog;
import org.apache.impala.thrift.TColumnValue;
import org.apache.impala.thrift.TDescribeOutputStyle;
import org.apache.impala.thrift.TDescribeResult;
+import org.apache.impala.thrift.TDescribeTableParams;
import org.apache.impala.thrift.TFunctionBinaryType;
import org.apache.impala.thrift.TPrivilege;
import org.apache.impala.thrift.TPrivilegeLevel;
@@ -272,8 +273,11 @@ public abstract class AuthorizationTestBase extends FrontendTestBase {
Preconditions.checkArgument(includedStrings_.length != 0 ||
excludedStrings_.length != 0,
"One or both of included or excluded strings must be defined.");
- List<String> result = resultToStringList(authzFrontend_.describeTable(table,
- outputStyle_, user_));
+ TDescribeTableParams testParams = new TDescribeTableParams();
+ testParams.setTable_name(table);
+ testParams.setOutput_style(outputStyle_);
+ List<String> result = resultToStringList(authzFrontend_.describeTable(testParams,
+ user_));
for (String str: includedStrings_) {
assertTrue(String.format("\"%s\" is not in the describe output.\n" +
"Expected : %s\n Actual : %s", str, Arrays.toString(includedStrings_),
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-metadata-tables.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-metadata-tables.test
index c34f3aacc..0f58dae99 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-metadata-tables.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-metadata-tables.test
@@ -420,9 +420,9 @@ INT,STRING,BIGINT
####
====
---- QUERY
-describe functional_parquet.iceberg_query_metadata.snapshots;
+describe formatted functional_parquet.iceberg_query_metadata.snapshots;
---- CATCH
-AnalysisException: Could not resolve path: 'functional_parquet.iceberg_query_metadata.snapshots'
+AnalysisException: DESCRIBE FORMATTED|EXTENDED cannot refer to a metadata table.
====
---- QUERY
show create table functional_parquet.iceberg_query_metadata.snapshots;
@@ -456,7 +456,7 @@ ParseException: Syntax error in line 1
====
####
-# Test 9 : Query STRUCT type columns
+# Test 9 : Query nested type columns
####
====
---- QUERY
@@ -550,4 +550,264 @@ as select equality_ids from functional_parquet.iceberg_query_metadata.all_files;
select item from iceberg_query_metadata_all_files a, a.equality_ids e, e.delete_ids;
---- CATCH
AnalysisException: Querying collection types (ARRAY/MAP) is not supported for Iceberg Metadata tables. (IMPALA-12610, IMPALA-12611)
-====
\ No newline at end of file
+====
+
+####
+# Test 10 : Describe all the metadata tables once
+####
+---- QUERY
+describe functional_parquet.iceberg_query_metadata.snapshots;
+---- RESULTS
+'committed_at','timestamp','','false'
+'snapshot_id','bigint','','false'
+'parent_id','bigint','','true'
+'operation','string','','true'
+'manifest_list','string','','true'
+'summary','map<string,string>','','true'
+---- TYPES
+STRING,STRING,STRING,STRING
+====
+---- QUERY
+describe functional_parquet.iceberg_query_metadata.`files`;
+---- RESULTS
+'content','int','Contents of the file: 0=data, 1=position deletes, 2=equality deletes','true'
+'file_path','string','Location URI with FS scheme','false'
+'file_format','string','File format name: avro, orc, or parquet','false'
+'spec_id','int','Partition spec ID','true'
+'record_count','bigint','Number of records in the file','false'
+'file_size_in_bytes','bigint','Total file size in bytes','false'
+'column_sizes','map<int,bigint>','Map of column id to total size on disk','true'
+'value_counts','map<int,bigint>','Map of column id to total count, including null and NaN','true'
+'null_value_counts','map<int,bigint>','Map of column id to null value count','true'
+'nan_value_counts','map<int,bigint>','Map of column id to number of NaN values in the column','true'
+'lower_bounds','map<int,binary>','Map of column id to lower bound','true'
+'upper_bounds','map<int,binary>','Map of column id to upper bound','true'
+'key_metadata','binary','Encryption key metadata blob','true'
+'split_offsets','array<bigint>','Splittable offsets','true'
+'equality_ids','array<int>','Equality comparison field IDs','true'
+'sort_order_id','int','Sort order ID','true'
+'readable_metrics','struct<\n i:struct<\n column_size:bigint comment ''total size on disk'',\n value_count:bigint comment ''total count, including null and nan'',\n null_value_count:bigint comment ''null value count'',\n nan_value_count:bigint comment ''nan value count'',\n lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper bound''\n > comment ''metrics for column i''\n>','Column metrics in readable form','true'
+---- TYPES
+STRING,STRING,STRING,STRING
+====
+---- QUERY
+describe functional_parquet.iceberg_query_metadata.data_files;
+---- RESULTS
+'content','int','Contents of the file: 0=data, 1=position deletes, 2=equality deletes','true'
+'file_path','string','Location URI with FS scheme','false'
+'file_format','string','File format name: avro, orc, or parquet','false'
+'spec_id','int','Partition spec ID','true'
+'record_count','bigint','Number of records in the file','false'
+'file_size_in_bytes','bigint','Total file size in bytes','false'
+'column_sizes','map<int,bigint>','Map of column id to total size on disk','true'
+'value_counts','map<int,bigint>','Map of column id to total count, including null and NaN','true'
+'null_value_counts','map<int,bigint>','Map of column id to null value count','true'
+'nan_value_counts','map<int,bigint>','Map of column id to number of NaN values in the column','true'
+'lower_bounds','map<int,binary>','Map of column id to lower bound','true'
+'upper_bounds','map<int,binary>','Map of column id to upper bound','true'
+'key_metadata','binary','Encryption key metadata blob','true'
+'split_offsets','array<bigint>','Splittable offsets','true'
+'equality_ids','array<int>','Equality comparison field IDs','true'
+'sort_order_id','int','Sort order ID','true'
+'readable_metrics','struct<\n i:struct<\n column_size:bigint comment ''total size on disk'',\n value_count:bigint comment ''total count, including null and nan'',\n null_value_count:bigint comment ''null value count'',\n nan_value_count:bigint comment ''nan value count'',\n lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper bound''\n > comment ''metrics for column i''\n>','Column metrics in readable form','true'
+---- TYPES
+STRING,STRING,STRING,STRING
+====
+---- QUERY
+describe functional_parquet.iceberg_query_metadata.delete_files;
+---- RESULTS
+'content','int','Contents of the file: 0=data, 1=position deletes, 2=equality deletes','true'
+'file_path','string','Location URI with FS scheme','false'
+'file_format','string','File format name: avro, orc, or parquet','false'
+'spec_id','int','Partition spec ID','true'
+'record_count','bigint','Number of records in the file','false'
+'file_size_in_bytes','bigint','Total file size in bytes','false'
+'column_sizes','map<int,bigint>','Map of column id to total size on disk','true'
+'value_counts','map<int,bigint>','Map of column id to total count, including null and NaN','true'
+'null_value_counts','map<int,bigint>','Map of column id to null value count','true'
+'nan_value_counts','map<int,bigint>','Map of column id to number of NaN values in the column','true'
+'lower_bounds','map<int,binary>','Map of column id to lower bound','true'
+'upper_bounds','map<int,binary>','Map of column id to upper bound','true'
+'key_metadata','binary','Encryption key metadata blob','true'
+'split_offsets','array<bigint>','Splittable offsets','true'
+'equality_ids','array<int>','Equality comparison field IDs','true'
+'sort_order_id','int','Sort order ID','true'
+'readable_metrics','struct<\n i:struct<\n column_size:bigint comment ''total size on disk'',\n value_count:bigint comment ''total count, including null and nan'',\n null_value_count:bigint comment ''null value count'',\n nan_value_count:bigint comment ''nan value count'',\n lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper bound''\n > comment ''metrics for column i''\n>','Column metrics in readable form','true'
+---- TYPES
+STRING,STRING,STRING,STRING
+====
+---- QUERY
+describe functional_parquet.iceberg_query_metadata.history;
+---- RESULTS
+'made_current_at','timestamp','','false'
+'snapshot_id','bigint','','false'
+'parent_id','bigint','','true'
+'is_current_ancestor','boolean','','false'
+---- TYPES
+STRING,STRING,STRING,STRING
+====
+---- QUERY
+describe functional_parquet.iceberg_query_metadata.metadata_log_entries;
+---- RESULTS
+'timestamp','timestamp','','false'
+'file','string','','false'
+'latest_snapshot_id','bigint','','true'
+'latest_schema_id','int','','true'
+'latest_sequence_number','bigint','','true'
+---- TYPES
+STRING,STRING,STRING,STRING
+====
+---- QUERY
+describe functional_parquet.iceberg_query_metadata.snapshots;
+---- RESULTS
+'committed_at','timestamp','','false'
+'snapshot_id','bigint','','false'
+'parent_id','bigint','','true'
+'operation','string','','true'
+'manifest_list','string','','true'
+'summary','map<string,string>','','true'
+---- TYPES
+STRING,STRING,STRING,STRING
+====
+---- QUERY
+describe functional_parquet.iceberg_query_metadata.refs;
+---- RESULTS
+'name','string','','false'
+'type','string','','false'
+'snapshot_id','bigint','','false'
+'max_reference_age_in_ms','bigint','','true'
+'min_snapshots_to_keep','int','','true'
+'max_snapshot_age_in_ms','bigint','','true'
+---- TYPES
+STRING,STRING,STRING,STRING
+====
+---- QUERY
+describe functional_parquet.iceberg_query_metadata.manifests;
+---- RESULTS
+'content','int','','false'
+'path','string','','false'
+'length','bigint','','false'
+'partition_spec_id','int','','false'
+'added_snapshot_id','bigint','','false'
+'added_data_files_count','int','','false'
+'existing_data_files_count','int','','false'
+'deleted_data_files_count','int','','false'
+'added_delete_files_count','int','','false'
+'existing_delete_files_count','int','','false'
+'deleted_delete_files_count','int','','false'
+'partition_summaries','array<struct<\n contains_null:boolean,\n contains_nan:boolean,\n lower_bound:string,\n upper_bound:string\n>>','','false'
+---- TYPES
+STRING,STRING,STRING,STRING
+====
+---- QUERY
+describe functional_parquet.iceberg_query_metadata.`partitions`;
+---- RESULTS
+'record_count','bigint','Count of records in data files','false'
+'file_count','int','Count of data files','false'
+'position_delete_record_count','bigint','Count of records in position delete files','false'
+'position_delete_file_count','int','Count of position delete files','false'
+'equality_delete_record_count','bigint','Count of records in equality delete files','false'
+'equality_delete_file_count','int','Count of equality delete files','false'
+---- TYPES
+STRING,STRING,STRING,STRING
+====
+---- QUERY
+describe functional_parquet.iceberg_query_metadata.all_data_files;
+---- RESULTS
+'content','int','Contents of the file: 0=data, 1=position deletes, 2=equality deletes','true'
+'file_path','string','Location URI with FS scheme','false'
+'file_format','string','File format name: avro, orc, or parquet','false'
+'spec_id','int','Partition spec ID','true'
+'record_count','bigint','Number of records in the file','false'
+'file_size_in_bytes','bigint','Total file size in bytes','false'
+'column_sizes','map<int,bigint>','Map of column id to total size on disk','true'
+'value_counts','map<int,bigint>','Map of column id to total count, including null and NaN','true'
+'null_value_counts','map<int,bigint>','Map of column id to null value count','true'
+'nan_value_counts','map<int,bigint>','Map of column id to number of NaN values in the column','true'
+'lower_bounds','map<int,binary>','Map of column id to lower bound','true'
+'upper_bounds','map<int,binary>','Map of column id to upper bound','true'
+'key_metadata','binary','Encryption key metadata blob','true'
+'split_offsets','array<bigint>','Splittable offsets','true'
+'equality_ids','array<int>','Equality comparison field IDs','true'
+'sort_order_id','int','Sort order ID','true'
+'readable_metrics','struct<\n i:struct<\n column_size:bigint comment ''total size on disk'',\n value_count:bigint comment ''total count, including null and nan'',\n null_value_count:bigint comment ''null value count'',\n nan_value_count:bigint comment ''nan value count'',\n lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper bound''\n > comment ''metrics for column i''\n>','Column metrics in readable form','true'
+---- TYPES
+STRING,STRING,STRING,STRING
+====
+---- QUERY
+describe functional_parquet.iceberg_query_metadata.all_delete_files;
+---- RESULTS
+'content','int','Contents of the file: 0=data, 1=position deletes, 2=equality deletes','true'
+'file_path','string','Location URI with FS scheme','false'
+'file_format','string','File format name: avro, orc, or parquet','false'
+'spec_id','int','Partition spec ID','true'
+'record_count','bigint','Number of records in the file','false'
+'file_size_in_bytes','bigint','Total file size in bytes','false'
+'column_sizes','map<int,bigint>','Map of column id to total size on disk','true'
+'value_counts','map<int,bigint>','Map of column id to total count, including null and NaN','true'
+'null_value_counts','map<int,bigint>','Map of column id to null value count','true'
+'nan_value_counts','map<int,bigint>','Map of column id to number of NaN values in the column','true'
+'lower_bounds','map<int,binary>','Map of column id to lower bound','true'
+'upper_bounds','map<int,binary>','Map of column id to upper bound','true'
+'key_metadata','binary','Encryption key metadata blob','true'
+'split_offsets','array<bigint>','Splittable offsets','true'
+'equality_ids','array<int>','Equality comparison field IDs','true'
+'sort_order_id','int','Sort order ID','true'
+'readable_metrics','struct<\n i:struct<\n column_size:bigint comment ''total size on disk'',\n value_count:bigint comment ''total count, including null and nan'',\n null_value_count:bigint comment ''null value count'',\n nan_value_count:bigint comment ''nan value count'',\n lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper bound''\n > comment ''metrics for column i''\n>','Column metrics in readable form','true'
+---- TYPES
+STRING,STRING,STRING,STRING
+====
+---- QUERY
+describe functional_parquet.iceberg_query_metadata.all_files;
+---- RESULTS
+'content','int','Contents of the file: 0=data, 1=position deletes, 2=equality deletes','true'
+'file_path','string','Location URI with FS scheme','false'
+'file_format','string','File format name: avro, orc, or parquet','false'
+'spec_id','int','Partition spec ID','true'
+'record_count','bigint','Number of records in the file','false'
+'file_size_in_bytes','bigint','Total file size in bytes','false'
+'column_sizes','map<int,bigint>','Map of column id to total size on disk','true'
+'value_counts','map<int,bigint>','Map of column id to total count, including null and NaN','true'
+'null_value_counts','map<int,bigint>','Map of column id to null value count','true'
+'nan_value_counts','map<int,bigint>','Map of column id to number of NaN values in the column','true'
+'lower_bounds','map<int,binary>','Map of column id to lower bound','true'
+'upper_bounds','map<int,binary>','Map of column id to upper bound','true'
+'key_metadata','binary','Encryption key metadata blob','true'
+'split_offsets','array<bigint>','Splittable offsets','true'
+'equality_ids','array<int>','Equality comparison field IDs','true'
+'sort_order_id','int','Sort order ID','true'
+'readable_metrics','struct<\n i:struct<\n column_size:bigint comment ''total size on disk'',\n value_count:bigint comment ''total count, including null and nan'',\n null_value_count:bigint comment ''null value count'',\n nan_value_count:bigint comment ''nan value count'',\n lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper bound''\n > comment ''metrics for column i''\n>','Column metrics in readable form','true'
+---- TYPES
+STRING,STRING,STRING,STRING
+====
+---- QUERY
+describe functional_parquet.iceberg_query_metadata.all_manifests;
+---- RESULTS
+'content','int','','false'
+'path','string','','false'
+'length','bigint','','false'
+'partition_spec_id','int','','true'
+'added_snapshot_id','bigint','','true'
+'added_data_files_count','int','','true'
+'existing_data_files_count','int','','true'
+'deleted_data_files_count','int','','true'
+'added_delete_files_count','int','','false'
+'existing_delete_files_count','int','','false'
+'deleted_delete_files_count','int','','false'
+'partition_summaries','array<struct<\n contains_null:boolean,\n contains_nan:boolean,\n lower_bound:string,\n upper_bound:string\n>>','','true'
+'reference_snapshot_id','bigint','','false'
+---- TYPES
+STRING,STRING,STRING,STRING
+====
+---- QUERY
+describe functional_parquet.iceberg_query_metadata.all_entries;
+---- RESULTS
+'status','int','','false'
+'snapshot_id','bigint','','true'
+'sequence_number','bigint','','true'
+'file_sequence_number','bigint','','true'
+'data_file','struct<\n content:int comment ''contents of the file: 0=data, 1=position deletes, 2=equality deletes'',\n file_path:string comment ''location uri with fs scheme'',\n file_format:string comment ''file format name: avro, orc, or parquet'',\n spec_id:int comment ''partition spec id'',\n record_count:bigint comment ''number of records in the file'',\n file_size_in_bytes:bigint comment ''total file size in bytes'',\n column_sizes:map<int,bigint> comment ''map of column id [...]
+'readable_metrics','struct<\n i:struct<\n column_size:bigint comment ''total size on disk'',\n value_count:bigint comment ''total count, including null and nan'',\n null_value_count:bigint comment ''null value count'',\n nan_value_count:bigint comment ''nan value count'',\n lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper bound''\n > comment ''metrics for column i''\n>','Column metrics in readable form','true'
+---- TYPES
+STRING,STRING,STRING,STRING
+====