You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by db...@apache.org on 2022/07/06 11:52:30 UTC
[impala] 02/02: IMPALA-11320: SHOW PARTITIONS on Iceberg table doesn't list the partitions
This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 4174ff5ea6a1b413a6218d26bbb7cf20809623cd
Author: gaoxq <ga...@gmail.com>
AuthorDate: Sat Jun 18 21:24:24 2022 +0800
IMPALA-11320: SHOW PARTITIONS on Iceberg table doesn't list the partitions
Currently, SHOW PARTITIONS on Iceberg tables only outputs the partition
spec which is not too useful.
Instead it should output the concrete partitions, number of files, number
of rows in each partitions. E.g.:
SHOW PARTITIONS ice_ctas_hadoop_tables_part;
'{"d_month":"613"}',4,2
'{"d_month":"614"}',3,1
'{"d_month":"615"}',2,1
Testing:
- Added end-to-end test
Change-Id: I3b4399ae924dadb89875735b12a2f92453b6754c
Reviewed-on: http://gerrit.cloudera.org:8080/18641
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
common/thrift/CatalogObjects.thrift | 6 ++
.../org/apache/impala/catalog/FeIcebergTable.java | 99 +++++++++++++++++-----
.../org/apache/impala/catalog/IcebergTable.java | 10 +++
.../impala/catalog/iceberg/IcebergCtasTarget.java | 6 ++
.../impala/catalog/local/LocalIcebergTable.java | 9 ++
.../java/org/apache/impala/service/Frontend.java | 2 +-
.../queries/QueryTest/iceberg-alter.test | 16 +---
.../queries/QueryTest/iceberg-create.test | 61 +------------
.../queries/QueryTest/iceberg-ctas.test | 59 +++++++++++++
.../QueryTest/iceberg-partitioned-insert.test | 86 +++++++++++++++++++
.../queries/QueryTest/iceberg-query.test | 20 +++--
11 files changed, 269 insertions(+), 105 deletions(-)
diff --git a/common/thrift/CatalogObjects.thrift b/common/thrift/CatalogObjects.thrift
index d0ae40bfd..f00bc6179 100644
--- a/common/thrift/CatalogObjects.thrift
+++ b/common/thrift/CatalogObjects.thrift
@@ -571,6 +571,11 @@ struct TIcebergPartitionSpec {
2: optional list<TIcebergPartitionField> partition_fields
}
+struct TIcebergPartitionStats {
+ 1: required i64 num_files;
+ 2: required i64 num_rows;
+}
+
struct TIcebergTable {
// Iceberg file system table location
1: required string table_location
@@ -589,6 +594,7 @@ struct TIcebergTable {
// properties
8: optional i64 parquet_plain_page_size;
9: optional i64 parquet_dict_page_size;
+ 10: optional map<string, TIcebergPartitionStats> partition_stats;
}
// Represents a table or view.
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java b/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java
index 52b722711..6ab595d8f 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java
@@ -22,10 +22,12 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
+import java.util.stream.Collectors;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
@@ -41,6 +43,7 @@ import org.apache.iceberg.Table;
import org.apache.impala.analysis.IcebergPartitionField;
import org.apache.impala.analysis.IcebergPartitionSpec;
import org.apache.impala.analysis.LiteralExpr;
+import org.apache.impala.analysis.PartitionKeyValue;
import org.apache.impala.catalog.HdfsPartition.FileDescriptor;
import org.apache.impala.common.FileSystemUtil;
import org.apache.impala.common.Reference;
@@ -53,6 +56,7 @@ import org.apache.impala.thrift.THdfsTable;
import org.apache.impala.thrift.THdfsPartition;
import org.apache.impala.thrift.TIcebergCatalog;
import org.apache.impala.thrift.TIcebergFileFormat;
+import org.apache.impala.thrift.TIcebergPartitionStats;
import org.apache.impala.thrift.TIcebergTable;
import org.apache.impala.thrift.TNetworkAddress;
import org.apache.impala.thrift.TResultSet;
@@ -60,6 +64,7 @@ import org.apache.impala.thrift.TResultSetMetadata;
import org.apache.impala.util.IcebergUtil;
import org.apache.impala.util.ListMap;
import org.apache.impala.util.TResultRowBuilder;
+import org.json.simple.JSONValue;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Ints;
@@ -77,6 +82,7 @@ public interface FeIcebergTable extends FeFsTable {
*/
Map<String, HdfsPartition.FileDescriptor> getPathHashToFileDescMap();
+ Map<String, TIcebergPartitionStats> getIcebergPartitionStats();
/**
* Return the hdfs table transformed from iceberg table
*/
@@ -294,33 +300,32 @@ public interface FeIcebergTable extends FeFsTable {
return format == HdfsFileFormat.PARQUET || format == HdfsFileFormat.ORC;
}
- public static TResultSet getPartitionSpecs(FeIcebergTable table)
+ public static TResultSet getPartitionStats(FeIcebergTable table)
throws TableLoadingException {
TResultSet result = new TResultSet();
TResultSetMetadata resultSchema = new TResultSetMetadata();
result.setSchema(resultSchema);
-
- resultSchema.addToColumns(new TColumn("Partition Id", Type.BIGINT.toThrift()));
- resultSchema.addToColumns(new TColumn("Source Id", Type.BIGINT.toThrift()));
- resultSchema.addToColumns(new TColumn("Field Id", Type.BIGINT.toThrift()));
- resultSchema.addToColumns(new TColumn("Field Name", Type.STRING.toThrift()));
- resultSchema.addToColumns(new TColumn("Field Partition Transform",
- Type.STRING.toThrift()));
-
- if (!table.getIcebergApiTable().specs().isEmpty()) {
- // Just show the current PartitionSpec from Iceberg table metadata
- PartitionSpec latestSpec = table.getIcebergApiTable().spec();
- HashMap<String, Integer> transformParams =
- IcebergUtil.getPartitionTransformParams(latestSpec);
- for(PartitionField field : latestSpec.fields()) {
- TResultRowBuilder builder = new TResultRowBuilder();
- builder.add(latestSpec.specId());
- builder.add(field.sourceId());
- builder.add(field.fieldId());
- builder.add(field.name());
- builder.add(IcebergUtil.getPartitionTransform(field, transformParams).toSql());
- result.addToRows(builder.get());
- }
+ result.setRows(new ArrayList<>());
+
+ resultSchema.addToColumns(new TColumn("Partition", Type.STRING.toThrift()));
+ resultSchema.addToColumns(new TColumn("Number Of Rows", Type.BIGINT.toThrift()));
+ resultSchema.addToColumns(new TColumn("Number Of Files", Type.BIGINT.toThrift()));
+
+ Map<String, TIcebergPartitionStats> fieldNameToPartitionValue =
+ table.getIcebergPartitionStats()
+ .entrySet()
+ .stream()
+ .sorted(Map.Entry.comparingByKey())
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,
+ (oldValue, newValue) -> oldValue, LinkedHashMap::new));
+
+ for (Map.Entry<String, TIcebergPartitionStats> partitionInfo :
+ fieldNameToPartitionValue.entrySet()) {
+ TResultRowBuilder builder = new TResultRowBuilder();
+ builder.add(partitionInfo.getKey());
+ builder.add(partitionInfo.getValue().getNum_rows());
+ builder.add(partitionInfo.getValue().getNum_files());
+ result.addToRows(builder.get());
}
return result;
}
@@ -453,6 +458,7 @@ public interface FeIcebergTable extends FeFsTable {
icebergTable.getIcebergParquetPlainPageSize());
tIcebergTable.setParquet_dict_page_size(
icebergTable.getIcebergParquetDictPageSize());
+ tIcebergTable.setPartition_stats(icebergTable.getIcebergPartitionStats());
return tIcebergTable;
}
@@ -565,6 +571,53 @@ public interface FeIcebergTable extends FeFsTable {
return fileDescMap;
}
+ /**
+ * Return the PartitionContent loaded by the internal HdfsTable. To avoid returning
+ * the metadata files the result set is limited to the files that are tracked by
+ * Iceberg. Both the number of rows and number of files show in partitionStats.
+ */
+ public static Map<String, TIcebergPartitionStats> loadPartitionStats(
+ IcebergTable table) throws TableLoadingException {
+ List<DataFile> dataFileList =
+ IcebergUtil
+ .getIcebergDataFiles(table, new ArrayList<>(), /*timeTravelSpecl=*/null)
+ .first;
+ Map<String, TIcebergPartitionStats> partitionStats = new HashMap<>();
+ for (DataFile dataFile : dataFileList) {
+ String partitionKey = getParitionKey(table, dataFile);
+ if (partitionStats.containsKey(partitionKey)) {
+ TIcebergPartitionStats info = partitionStats.get(partitionKey);
+ info.num_rows += dataFile.recordCount();
+ info.num_files += 1;
+ partitionStats.put(partitionKey, info);
+ } else {
+ TIcebergPartitionStats icebergPartitionStats =
+ new TIcebergPartitionStats();
+ icebergPartitionStats.num_rows = dataFile.recordCount();
+ icebergPartitionStats.num_files = 1;
+ partitionStats.put(partitionKey, icebergPartitionStats);
+ }
+ }
+ return partitionStats;
+ }
+
+ /**
+ * Get iceberg partition from a dataFile and wrapper to a json string
+ */
+ public static String getParitionKey(IcebergTable table, DataFile dataFile) {
+ PartitionSpec spec = table.getIcebergApiTable().specs().get(dataFile.specId());
+ Map<String, String> fieldNameToPartitionValue = new LinkedHashMap<>();
+ for (int i = 0; i < spec.fields().size(); ++i) {
+ Object partValue = dataFile.partition().get(i, Object.class);
+ String partValueString = null;
+ if (partValue != null) {
+ partValueString = partValue.toString();
+ }
+ fieldNameToPartitionValue.put(spec.fields().get(i).name(), partValueString);
+ }
+ return JSONValue.toJSONString(fieldNameToPartitionValue);
+ }
+
/**
* Get iceberg partition spec by iceberg table metadata
*/
diff --git a/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java b/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
index 523dcbac8..b5da61a19 100644
--- a/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
@@ -41,6 +41,7 @@ import org.apache.impala.thrift.TIcebergCatalog;
import org.apache.impala.thrift.TIcebergFileFormat;
import org.apache.impala.thrift.TIcebergPartitionField;
import org.apache.impala.thrift.TIcebergPartitionSpec;
+import org.apache.impala.thrift.TIcebergPartitionStats;
import org.apache.impala.thrift.TIcebergTable;
import org.apache.impala.thrift.TPartialPartitionInfo;
import org.apache.impala.thrift.TTable;
@@ -164,6 +165,8 @@ public class IcebergTable extends Table implements FeIcebergTable {
// The snapshot id cached in the CatalogD, necessary to syncronize the caches.
private long catalogSnapshotId_ = -1;
+ private Map<String, TIcebergPartitionStats> partitionStats_;
+
protected IcebergTable(org.apache.hadoop.hive.metastore.api.Table msTable,
Db db, String name, String owner) {
super(msTable, db, name, owner);
@@ -302,6 +305,11 @@ public class IcebergTable extends Table implements FeIcebergTable {
return catalogSnapshotId_;
}
+ @Override
+ public Map<String, TIcebergPartitionStats> getIcebergPartitionStats() {
+ return partitionStats_;
+ }
+
@Override
public TTable toThrift() {
TTable table = super.toThrift();
@@ -345,6 +353,7 @@ public class IcebergTable extends Table implements FeIcebergTable {
hdfsTable_
.load(false, msClient, msTable_, true, true, false, null, null,null, reason);
pathHashToFileDescMap_ = Utils.loadAllPartition(this);
+ partitionStats_ = Utils.loadPartitionStats(this);
loadAllColumnStats(msClient);
} catch (Exception e) {
throw new IcebergTableLoadingException("Error loading metadata for Iceberg table "
@@ -429,6 +438,7 @@ public class IcebergTable extends Table implements FeIcebergTable {
pathHashToFileDescMap_ = FeIcebergTable.Utils.loadFileDescMapFromThrift(
ticeberg.getPath_hash_to_file_descriptor(), null, null);
hdfsTable_.loadFromThrift(thriftTable);
+ partitionStats_ = ticeberg.getPartition_stats();
}
private List<IcebergPartitionSpec> loadPartitionBySpecsFromThrift(
diff --git a/fe/src/main/java/org/apache/impala/catalog/iceberg/IcebergCtasTarget.java b/fe/src/main/java/org/apache/impala/catalog/iceberg/IcebergCtasTarget.java
index 53e3fe6e3..fe98ce486 100644
--- a/fe/src/main/java/org/apache/impala/catalog/iceberg/IcebergCtasTarget.java
+++ b/fe/src/main/java/org/apache/impala/catalog/iceberg/IcebergCtasTarget.java
@@ -63,6 +63,7 @@ import org.apache.impala.thrift.THdfsStorageDescriptor;
import org.apache.impala.thrift.THdfsTable;
import org.apache.impala.thrift.TIcebergCatalog;
import org.apache.impala.thrift.TIcebergFileFormat;
+import org.apache.impala.thrift.TIcebergPartitionStats;
import org.apache.impala.thrift.TImpalaTableType;
import org.apache.impala.thrift.TTableDescriptor;
import org.apache.impala.thrift.TTableType;
@@ -256,6 +257,11 @@ public class IcebergCtasTarget extends CtasTargetTable implements FeIcebergTable
return -1;
}
+ @Override
+ public Map<String, TIcebergPartitionStats> getIcebergPartitionStats() {
+ return null;
+ }
+
@Override
public org.apache.iceberg.Table getIcebergApiTable() {
return null;
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalIcebergTable.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalIcebergTable.java
index dd78538bc..c128f4251 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalIcebergTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalIcebergTable.java
@@ -40,6 +40,7 @@ import org.apache.impala.thrift.THdfsPartition;
import org.apache.impala.thrift.THdfsTable;
import org.apache.impala.thrift.TIcebergCatalog;
import org.apache.impala.thrift.TIcebergFileFormat;
+import org.apache.impala.thrift.TIcebergPartitionStats;
import org.apache.impala.thrift.TPartialTableInfo;
import org.apache.impala.thrift.TTableDescriptor;
import org.apache.impala.thrift.TTableType;
@@ -73,6 +74,8 @@ public class LocalIcebergTable extends LocalTable implements FeIcebergTable {
// Cached Iceberg API table object.
private org.apache.iceberg.Table icebergApiTable_;
+ private Map<String, TIcebergPartitionStats> partitionStats_;
+
/**
* Loads the Iceberg metadata from the CatalogD then initializes a LocalIcebergTable.
*/
@@ -123,6 +126,7 @@ public class LocalIcebergTable extends LocalTable implements FeIcebergTable {
icebergParquetRowGroupSize_ = Utils.getIcebergParquetRowGroupSize(msTable);
icebergParquetPlainPageSize_ = Utils.getIcebergParquetPlainPageSize(msTable);
icebergParquetDictPageSize_ = Utils.getIcebergParquetDictPageSize(msTable);
+ partitionStats_ = tableInfo.getIceberg_table().getPartition_stats();
addVirtualColumns();
}
@@ -213,6 +217,11 @@ public class LocalIcebergTable extends LocalTable implements FeIcebergTable {
return catalogSnapshotId_;
}
+ @Override
+ public Map<String, TIcebergPartitionStats> getIcebergPartitionStats() {
+ return partitionStats_;
+ }
+
@Override
public TTableDescriptor toThriftDescriptor(int tableId,
Set<Long> referencedPartitions) {
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index 85e3ed03d..9d5b12f86 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -1433,7 +1433,7 @@ public class Frontend {
FeTable table = getCatalog().getTable(dbName, tableName);
if (table instanceof FeFsTable) {
if (table instanceof FeIcebergTable && op == TShowStatsOp.PARTITIONS) {
- return FeIcebergTable.Utils.getPartitionSpecs((FeIcebergTable) table);
+ return FeIcebergTable.Utils.getPartitionStats((FeIcebergTable) table);
}
return ((FeFsTable) table).getTableStats();
} else if (table instanceof FeHBaseTable) {
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test
index b715a2254..2438f5602 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test
@@ -65,14 +65,8 @@ ALTER TABLE iceberg_hadoop_tables SET PARTITION SPEC (
price);
SHOW PARTITIONS iceberg_hadoop_tables;
---- RESULTS
-regex:.+,1,1000,'level','IDENTITY'
-regex:.+,1,1001,'level_trunc','TRUNCATE[5]'
-regex:.+,2,1002,'event_time_hour','HOUR'
-regex:.+,3,1003,'register_time_year','YEAR'
-regex:.+,4,1004,'message_bucket','BUCKET[15]'
-regex:.+,5,1005,'price','IDENTITY'
---- TYPES
-BIGINT, BIGINT, BIGINT, STRING, STRING
+STRING, BIGINT, BIGINT
====
---- QUERY
CREATE TABLE iceberg_hadoop_catalog(
@@ -132,14 +126,8 @@ ALTER TABLE iceberg_hadoop_catalog SET PARTITION SPEC (
price);
SHOW PARTITIONS iceberg_hadoop_catalog;
---- RESULTS
-regex:.+,1,1000,'level','IDENTITY'
-regex:.+,1,1001,'level_trunc','TRUNCATE[5]'
-regex:.+,2,1002,'event_time_hour','HOUR'
-regex:.+,3,1003,'register_time_year','YEAR'
-regex:.+,4,1004,'message_bucket','BUCKET[15]'
-regex:.+,5,1005,'price','IDENTITY'
---- TYPES
-BIGINT, BIGINT, BIGINT, STRING, STRING
+STRING, BIGINT, BIGINT
====
---- QUERY
CREATE TABLE iceberg_rename (i int)
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test
index cf083af47..a4a7223a0 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test
@@ -39,14 +39,8 @@ STRING,STRING,STRING,STRING
---- QUERY
SHOW PARTITIONS iceberg_hadoop_tables;
---- RESULTS
-0,1,1000,'level','IDENTITY'
-0,1,1001,'level_trunc','TRUNCATE[10]'
-0,2,1002,'event_time','IDENTITY'
-0,2,1003,'event_time_hour','HOUR'
-0,2,1004,'event_time_bucket','BUCKET[1000]'
-0,3,1005,'register_time_day','DAY'
---- TYPES
-BIGINT,BIGINT,BIGINT,STRING,STRING
+STRING, BIGINT,BIGINT
====
---- QUERY
DROP TABLE iceberg_hadoop_tables;
@@ -97,11 +91,8 @@ STRING,STRING,STRING,STRING
---- QUERY
SHOW PARTITIONS iceberg_hadoop_tbls_external;
---- RESULTS
-0,1,1000,'level','IDENTITY'
-0,1,1001,'level_bucket','BUCKET[12345]'
-0,1,1002,'level_trunc','TRUNCATE[15]'
---- TYPES
-BIGINT,BIGINT,BIGINT,STRING,STRING
+STRING,BIGINT,BIGINT
====
---- QUERY
CREATE EXTERNAL TABLE iceberg_hadoop_tbls_external_empty_col
@@ -119,15 +110,6 @@ DESCRIBE iceberg_hadoop_tbls_external_empty_col;
STRING,STRING,STRING,STRING
====
---- QUERY
-SHOW PARTITIONS iceberg_hadoop_tbls_external_empty_col;
----- RESULTS
-0,1,1000,'level','IDENTITY'
-0,1,1001,'level_bucket','BUCKET[12345]'
-0,1,1002,'level_trunc','TRUNCATE[15]'
----- TYPES
-BIGINT,BIGINT,BIGINT,STRING,STRING
-====
----- QUERY
DROP TABLE iceberg_hadoop_tbls_with_loc;
DROP TABLE iceberg_hadoop_tbls_external;
DROP TABLE iceberg_hadoop_tbls_external_empty_col;
@@ -173,18 +155,6 @@ DESCRIBE iceberg_hadoop_catalog;
STRING,STRING,STRING,STRING
====
---- QUERY
-SHOW PARTITIONS iceberg_hadoop_catalog;
----- RESULTS
-0,1,1000,'level','IDENTITY'
-0,1,1001,'level_trunc','TRUNCATE[10]'
-0,2,1002,'event_time','IDENTITY'
-0,2,1003,'event_time_hour','HOUR'
-0,2,1004,'event_time_bucket','BUCKET[1000]'
-0,3,1005,'register_time_day','DAY'
----- TYPES
-BIGINT,BIGINT,BIGINT,STRING,STRING
-====
----- QUERY
DROP TABLE iceberg_hadoop_catalog;
---- RESULTS
'Table has been dropped.'
@@ -219,15 +189,6 @@ DESCRIBE iceberg_hadoop_cat_external;
STRING,STRING,STRING,STRING
====
---- QUERY
-SHOW PARTITIONS iceberg_hadoop_cat_external;
----- RESULTS
-0,1,1000,'level','IDENTITY'
-0,1,1001,'level_bucket','BUCKET[12345]'
-0,1,1002,'level_trunc','TRUNCATE[10]'
----- TYPES
-BIGINT,BIGINT,BIGINT,STRING,STRING
-====
----- QUERY
DROP TABLE iceberg_hadoop_catalog;
DROP TABLE iceberg_hadoop_cat_external;
---- RESULTS
@@ -345,16 +306,6 @@ DESCRIBE iceberg_hadoop_cat_with_orc;
STRING,STRING,STRING,STRING
====
---- QUERY
-SHOW PARTITIONS iceberg_hadoop_cat_with_orc;
----- RESULTS
-0,1,1000,'level','IDENTITY'
-0,2,1001,'event_time','IDENTITY'
-0,2,1002,'event_time_hour','HOUR'
-0,3,1003,'register_time_day','DAY'
----- TYPES
-BIGINT,BIGINT,BIGINT,STRING,STRING
-====
----- QUERY
DROP TABLE iceberg_hadoop_cat_with_orc;
---- RESULTS
'Table has been dropped.'
@@ -466,14 +417,6 @@ DESCRIBE iceberg_old_style_partitions;
STRING,STRING,STRING,STRING
====
---- QUERY
-SHOW PARTITIONS iceberg_old_style_partitions;
----- RESULTS
-0,6,1000,'level','IDENTITY'
-0,7,1001,'event_id','IDENTITY'
----- TYPES
-BIGINT,BIGINT,BIGINT,STRING,STRING
-====
----- QUERY
create table ice_part_hadoop_tbl (
col_year date,
col_month date,
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-ctas.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-ctas.test
index 4102a521c..541fd4a39 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-ctas.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-ctas.test
@@ -37,6 +37,13 @@ row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/ice_ctas_part/data/d=2021-02-26
STRING, STRING, STRING
====
---- QUERY
+SHOW PARTITIONS ice_ctas_part;
+---- RESULTS
+'{"d":"18684"}',1,1
+---- TYPES
+STRING,BIGINT,BIGINT
+====
+---- QUERY
# INSERT an additional row to a different partition.
INSERT INTO ice_ctas_part VALUES ('fox','2021-02-27 16:16:59','2021-02-27');
SELECT * FROM ice_ctas_part;
@@ -55,6 +62,14 @@ row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/ice_ctas_part/data/d=2021-02-27
STRING, STRING, STRING
====
---- QUERY
+SHOW PARTITIONS ice_ctas_part;
+---- RESULTS
+'{"d":"18684"}',1,1
+'{"d":"18685"}',1,1
+---- TYPES
+STRING,BIGINT,BIGINT
+====
+---- QUERY
# Test CTAS in Iceberg HadoopTables catalog.
# Set table location to custom location.
# Use PARTITIONED BY SPEC
@@ -76,6 +91,41 @@ row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/loc_test/data/d_month=2021-02/.
STRING, STRING, STRING
====
---- QUERY
+SHOW PARTITIONS ice_ctas_hadoop_tables_part;
+---- RESULTS
+'{"d_month":"613"}',1,1
+---- TYPES
+STRING,BIGINT,BIGINT
+====
+---- QUERY
+INSERT INTO ice_ctas_hadoop_tables_part VALUES
+('fox','2021-02-27 16:16:59','2021-02-27'),('fox','2021-02-27 16:16:59','2021-02-27'),('cow','2021-02-28 17:16:59','2021-02-28'),
+('fox','2021-03-27 16:16:59','2021-03-27'),('fox','2021-03-27 16:16:59','2021-03-27'),('cow','2021-03-28 17:16:59','2021-03-28'),
+('fox','2021-04-27 16:16:59','2021-04-27'),('fox','2021-04-27 16:16:59','2021-04-27');
+SELECT * FROM ice_ctas_hadoop_tables_part;
+---- RESULTS
+'cow',2021-02-28 17:16:59,2021-02-28
+'cow',2021-03-28 17:16:59,2021-03-28
+'impala',2021-02-26 16:16:59,2021-02-26
+'fox',2021-02-27 16:16:59,2021-02-27
+'fox',2021-02-27 16:16:59,2021-02-27
+'fox',2021-03-27 16:16:59,2021-03-27
+'fox',2021-03-27 16:16:59,2021-03-27
+'fox',2021-04-27 16:16:59,2021-04-27
+'fox',2021-04-27 16:16:59,2021-04-27
+---- TYPES
+STRING,TIMESTAMP,DATE
+====
+---- QUERY
+SHOW PARTITIONS ice_ctas_hadoop_tables_part;
+---- RESULTS
+'{"d_month":"613"}',4,2
+'{"d_month":"614"}',3,1
+'{"d_month":"615"}',2,1
+---- TYPES
+STRING,BIGINT,BIGINT
+====
+---- QUERY
# Test CTAS in Iceberg HadoopCatalog catalog.
# Set 'iceberg.catalog_location' and 'iceberg.table_identifier'
# Partitioned by TRUNCATE
@@ -103,3 +153,12 @@ row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/cat_loc/ns1/ns2/ctas/data/s_tru
---- TYPES
STRING, STRING, STRING
====
+---- QUERY
+SHOW PARTITIONS ice_ctas_hadoop_catalog_part;
+---- RESULTS
+'{"s_trunc":"imp"}',1,1
+'{"s_trunc":"lio"}',1,1
+---- TYPES
+STRING,BIGINT,BIGINT
+====
+
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert.test
index 41c070109..fa0c7b413 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert.test
@@ -36,6 +36,17 @@ row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/ice_only_part/data/i=__HIVE_DEF
STRING, STRING, STRING
====
---- QUERY
+SHOW PARTITIONS ice_only_part;
+---- RESULTS
+'{"i":"1"}',1,1
+'{"i":"2"}',1,1
+'{"i":"3"}',1,1
+'{"i":"4"}',1,1
+'{"i":null}',1,1
+---- TYPES
+STRING, BIGINT, BIGINT
+====
+---- QUERY
# Test partition pruning with RUNTIME_PROFILE.
select * from ice_only_part
where i = 1;
@@ -157,6 +168,20 @@ row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/alltypes_part/data/id=7/bool_co
STRING, STRING, STRING
====
---- QUERY
+SHOW PARTITIONS alltypes_part;
+---- RESULTS
+'{"id":"0","bool_col":"true","int_col":"0","bigint_col":"0","float_col":"0.0","double_col":"0.0","date_col":"14245","string_col":"0"}',1,1
+'{"id":"1","bool_col":"false","int_col":"1","bigint_col":"10","float_col":"1.1","double_col":"10.1","date_col":"14245","string_col":"1"}',1,1
+'{"id":"2","bool_col":"true","int_col":"0","bigint_col":"0","float_col":"0.0","double_col":"0.0","date_col":"14276","string_col":"0"}',1,1
+'{"id":"3","bool_col":"false","int_col":"1","bigint_col":"10","float_col":"1.1","double_col":"10.1","date_col":"14276","string_col":"1"}',1,1
+'{"id":"4","bool_col":"true","int_col":"0","bigint_col":"0","float_col":"0.0","double_col":"0.0","date_col":"14304","string_col":"0"}',1,1
+'{"id":"5","bool_col":"false","int_col":"1","bigint_col":"10","float_col":"1.1","double_col":"10.1","date_col":"14304","string_col":"1"}',1,1
+'{"id":"6","bool_col":"true","int_col":"0","bigint_col":"0","float_col":"0.0","double_col":"0.0","date_col":"14335","string_col":"0"}',1,1
+'{"id":"7","bool_col":"false","int_col":"1","bigint_col":"10","float_col":"1.1","double_col":"10.1","date_col":"14335","string_col":"1"}',1,1
+---- TYPES
+STRING, BIGINT, BIGINT
+====
+---- QUERY
# INSERTs with wrong value orderings are rejected.
insert into alltypes_part
select bool_col, id, int_col, bigint_col, float_col, double_col,
@@ -269,6 +294,14 @@ row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/ice_part_non_order/data/s=secon
STRING, STRING, STRING
====
---- QUERY
+SHOW PARTITIONS ice_part_non_order;
+---- RESULTS
+'{"s":"first","d":"18603"}',1,1
+'{"s":"second","d":"18604"}',1,1
+---- TYPES
+STRING, BIGINT, BIGINT
+====
+---- QUERY
# Test inserts with different partition specs.
create table ice_alter_part (i int, d date, s string)
stored as iceberg;
@@ -335,6 +368,17 @@ row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/ice_alter_part/data/d_year=2020
STRING, STRING, STRING
====
---- QUERY
+SHOW PARTITIONS ice_alter_part;
+---- RESULTS
+'{"d_year":"50","i":"5","s_bucket":"2"}',1,1
+'{"d_year":"50","i":"6","s_bucket":"0"}',1,1
+'{"i":"3","d":"18605"}',1,1
+'{"i":"4","d":"18606"}',1,1
+'{}',2,1
+---- TYPES
+STRING, BIGINT, BIGINT
+====
+---- QUERY
create table ice_void (i int, s string, d date)
partitioned by spec (void(i), truncate(1, s), year(d))
stored as iceberg;
@@ -363,6 +407,15 @@ row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/ice_void/data/s_trunc=t/d_year=
STRING, STRING, STRING
====
---- QUERY
+SHOW PARTITIONS ice_void;
+---- RESULTS
+'{"i_null":null,"s_trunc":"o","d_year":"31"}',2,1
+'{"i_null":null,"s_trunc":"t","d_year":"32"}',1,1
+'{"i_null":null,"s_trunc":"t","d_year":"33"}',1,1
+---- TYPES
+STRING, BIGINT, BIGINT
+====
+---- QUERY
alter table ice_void set partition spec (i, void(s), year(d));
insert into ice_void values (4, 'four', '2004-04-04');
select * from ice_void;
@@ -388,6 +441,16 @@ row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/ice_void/data/i=4/d_year=2004/.
STRING, STRING, STRING
====
---- QUERY
+SHOW PARTITIONS ice_void;
+---- RESULTS
+'{"i":"4","s_null":null,"d_year":"34"}',1,1
+'{"i_null":null,"s_trunc":"o","d_year":"31"}',2,1
+'{"i_null":null,"s_trunc":"t","d_year":"32"}',1,1
+'{"i_null":null,"s_trunc":"t","d_year":"33"}',1,1
+---- TYPES
+STRING, BIGINT, BIGINT
+====
+---- QUERY
alter table ice_void set partition spec (i, void(s), void(d));
insert into ice_void values (5, 'five', '2005-05-05'), (5, 'other five', '1995-05-05');
select * from ice_void;
@@ -416,6 +479,17 @@ row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/ice_void/data/i=5/[^=/]*.0.parq
STRING, STRING, STRING
====
---- QUERY
+SHOW PARTITIONS ice_void;
+---- RESULTS
+'{"i":"4","s_null":null,"d_year":"34"}',1,1
+'{"i":"5","s_null":null,"d_null":null}',2,1
+'{"i_null":null,"s_trunc":"o","d_year":"31"}',2,1
+'{"i_null":null,"s_trunc":"t","d_year":"32"}',1,1
+'{"i_null":null,"s_trunc":"t","d_year":"33"}',1,1
+---- TYPES
+STRING, BIGINT, BIGINT
+====
+---- QUERY
alter table ice_void set partition spec (void(i), void(s), void(d));
insert into ice_void values (6, 'six', '2006-06-06'), (7, 'seven', '2007-07-07');
select * from ice_void;
@@ -447,6 +521,18 @@ row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/ice_void/data/[^=/]*.0.parq','.
STRING, STRING, STRING
====
---- QUERY
+SHOW PARTITIONS ice_void;
+---- RESULTS
+'{"i":"4","s_null":null,"d_year":"34"}',1,1
+'{"i":"5","s_null":null,"d_null":null}',2,1
+'{"i_null":null,"s_null":null,"d_null":null}',2,1
+'{"i_null":null,"s_trunc":"o","d_year":"31"}',2,1
+'{"i_null":null,"s_trunc":"t","d_year":"32"}',1,1
+'{"i_null":null,"s_trunc":"t","d_year":"33"}',1,1
+---- TYPES
+STRING, BIGINT, BIGINT
+====
+---- QUERY
create table store_sales partitioned by spec (ss_sold_date_sk) stored as iceberg
as select * from tpcds_parquet.store_sales;
select count(*) from store_sales;
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test
index 91b0144c2..9a5bfc473 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test
@@ -116,10 +116,11 @@ int,string,string,timestamp
# Test show partitions query for iceberg table
SHOW PARTITIONS iceberg_partitioned
---- TYPES
-bigint,bigint,bigint,string,string
+string,bigint,bigint
---- RESULTS
-0,4,1000,'event_time_hour','HOUR'
-0,3,1001,'action','IDENTITY'
+'{"event_time_hour":"438296","action":"view"}',8,8
+'{"event_time_hour":"438297","action":"click"}',6,6
+'{"event_time_hour":"438298","action":"download"}',6,6
====
---- QUERY
SHOW FILES in iceberg_partitioned
@@ -233,10 +234,11 @@ int,string,string,timestamp
---- QUERY
SHOW PARTITIONS hadoop_catalog_test_external
---- TYPES
-bigint,bigint,bigint,string,string
+string,bigint,bigint
---- RESULTS
-0,4,1000,'event_time_hour','HOUR'
-0,3,1001,'action','IDENTITY'
+'{"event_time_hour":"438296","action":"view"}',8,8
+'{"event_time_hour":"438297","action":"click"}',6,6
+'{"event_time_hour":"438298","action":"download"}',6,6
====
---- QUERY
SHOW FILES in hadoop_catalog_test_external
@@ -312,9 +314,11 @@ int,string,string
---- QUERY
SHOW PARTITIONS iceberg_partitioned_orc_external
---- TYPES
-bigint,bigint,bigint,string,string
+string,bigint,bigint
---- RESULTS
-0,3,1000,'action','IDENTITY'
+'{"action":"click"}',6,6
+'{"action":"download"}',6,6
+'{"action":"view"}',8,8
====
---- QUERY
SHOW FILES in iceberg_partitioned_orc_external