You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by et...@apache.org on 2025/02/25 07:08:48 UTC
(iceberg) branch main updated: Arrow, Parquet, Spark 3.5, Flink 1.20: Avoid deprecated method (#11874)
This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new a15f4710b2 Arrow, Parquet, Spark 3.5, Flink 1.20: Avoid deprecated method (#11874)
a15f4710b2 is described below
commit a15f4710b246f356eee5ae3594e6e758d88556b0
Author: Yuya Ebihara <eb...@gmail.com>
AuthorDate: Tue Feb 25 16:08:43 2025 +0900
Arrow, Parquet, Spark 3.5, Flink 1.20: Avoid deprecated method (#11874)
---
.../GenericArrowVectorAccessorFactory.java | 6 +-
.../arrow/vectorized/VectorizedArrowReader.java | 5 +-
.../flink/data/ParquetWithFlinkSchemaVisitor.java | 190 ++++++++++-----------
.../apache/iceberg/parquet/ParquetTypeVisitor.java | 18 +-
.../org/apache/iceberg/spark/SparkTableUtil.java | 2 +-
.../spark/data/ParquetWithSparkSchemaVisitor.java | 188 ++++++++++----------
.../iceberg/spark/TestSparkCatalogOperations.java | 6 +-
.../spark/source/TestCompressionSettings.java | 5 +-
8 files changed, 202 insertions(+), 218 deletions(-)
diff --git a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/GenericArrowVectorAccessorFactory.java b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/GenericArrowVectorAccessorFactory.java
index 6350660d57..ecbc1cf1d9 100644
--- a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/GenericArrowVectorAccessorFactory.java
+++ b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/GenericArrowVectorAccessorFactory.java
@@ -48,7 +48,7 @@ import org.apache.iceberg.parquet.ParquetUtil;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.column.Dictionary;
-import org.apache.parquet.schema.OriginalType;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.PrimitiveType;
/**
@@ -225,7 +225,9 @@ public class GenericArrowVectorAccessorFactory<
}
private static boolean isDecimal(PrimitiveType primitive) {
- return primitive != null && OriginalType.DECIMAL.equals(primitive.getOriginalType());
+ return primitive != null
+ && primitive.getLogicalTypeAnnotation()
+ instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
}
private static class BooleanAccessor<
diff --git a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorizedArrowReader.java b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorizedArrowReader.java
index f65774fec2..c71c8b1df5 100644
--- a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorizedArrowReader.java
+++ b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorizedArrowReader.java
@@ -50,7 +50,7 @@ import org.apache.parquet.column.Dictionary;
import org.apache.parquet.column.page.PageReadStore;
import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
import org.apache.parquet.hadoop.metadata.ColumnPath;
-import org.apache.parquet.schema.OriginalType;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.PrimitiveType;
/**
@@ -226,7 +226,8 @@ public class VectorizedArrowReader implements VectorizedReader<VectorHolder> {
PrimitiveType primitive = desc.getPrimitiveType();
PrimitiveType.PrimitiveTypeName typeName = primitive.getPrimitiveTypeName();
Types.NestedField physicalType = logicalType;
- if (OriginalType.DECIMAL.equals(primitive.getOriginalType())) {
+ if (primitive.getLogicalTypeAnnotation()
+ instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) {
org.apache.iceberg.types.Type type;
if (PrimitiveType.PrimitiveTypeName.INT64.equals(typeName)) {
// Use BigIntVector for long backed decimal
diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/data/ParquetWithFlinkSchemaVisitor.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/data/ParquetWithFlinkSchemaVisitor.java
index 33feb2e321..4770cb6f5f 100644
--- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/data/ParquetWithFlinkSchemaVisitor.java
+++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/data/ParquetWithFlinkSchemaVisitor.java
@@ -29,8 +29,8 @@ import org.apache.iceberg.avro.AvroSchemaUtil;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
-import org.apache.parquet.schema.OriginalType;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.Type;
@@ -51,106 +51,100 @@ public class ParquetWithFlinkSchemaVisitor<T> {
} else {
// if not a primitive, the typeId must be a group
GroupType group = type.asGroupType();
- OriginalType annotation = group.getOriginalType();
- if (annotation != null) {
- switch (annotation) {
- case LIST:
- Preconditions.checkArgument(
- !group.isRepetition(Type.Repetition.REPEATED),
- "Invalid list: top-level group is repeated: %s",
- group);
- Preconditions.checkArgument(
- group.getFieldCount() == 1,
- "Invalid list: does not contain single repeated field: %s",
- group);
-
- GroupType repeatedElement = group.getFields().get(0).asGroupType();
- Preconditions.checkArgument(
- repeatedElement.isRepetition(Type.Repetition.REPEATED),
- "Invalid list: inner group is not repeated");
- Preconditions.checkArgument(
- repeatedElement.getFieldCount() <= 1,
- "Invalid list: repeated group is not a single field: %s",
- group);
-
- Preconditions.checkArgument(
- sType instanceof ArrayType, "Invalid list: %s is not an array", sType);
- ArrayType array = (ArrayType) sType;
- RowType.RowField element =
- new RowField(
- "element", array.getElementType(), "element of " + array.asSummaryString());
-
- visitor.fieldNames.push(repeatedElement.getName());
- try {
- T elementResult = null;
- if (repeatedElement.getFieldCount() > 0) {
- elementResult = visitField(element, repeatedElement.getType(0), visitor);
- }
-
- return visitor.list(array, group, elementResult);
-
- } finally {
- visitor.fieldNames.pop();
- }
-
- case MAP:
- Preconditions.checkArgument(
- !group.isRepetition(Type.Repetition.REPEATED),
- "Invalid map: top-level group is repeated: %s",
- group);
- Preconditions.checkArgument(
- group.getFieldCount() == 1,
- "Invalid map: does not contain single repeated field: %s",
- group);
-
- GroupType repeatedKeyValue = group.getType(0).asGroupType();
- Preconditions.checkArgument(
- repeatedKeyValue.isRepetition(Type.Repetition.REPEATED),
- "Invalid map: inner group is not repeated");
- Preconditions.checkArgument(
- repeatedKeyValue.getFieldCount() <= 2,
- "Invalid map: repeated group does not have 2 fields");
-
- Preconditions.checkArgument(
- sType instanceof MapType, "Invalid map: %s is not a map", sType);
- MapType map = (MapType) sType;
- RowField keyField =
- new RowField("key", map.getKeyType(), "key of " + map.asSummaryString());
- RowField valueField =
- new RowField("value", map.getValueType(), "value of " + map.asSummaryString());
-
- visitor.fieldNames.push(repeatedKeyValue.getName());
- try {
- T keyResult = null;
- T valueResult = null;
- switch (repeatedKeyValue.getFieldCount()) {
- case 2:
- // if there are 2 fields, both key and value are projected
- keyResult = visitField(keyField, repeatedKeyValue.getType(0), visitor);
- valueResult = visitField(valueField, repeatedKeyValue.getType(1), visitor);
- break;
- case 1:
- // if there is just one, use the name to determine what it is
- Type keyOrValue = repeatedKeyValue.getType(0);
- if (keyOrValue.getName().equalsIgnoreCase("key")) {
- keyResult = visitField(keyField, keyOrValue, visitor);
- // value result remains null
- } else {
- valueResult = visitField(valueField, keyOrValue, visitor);
- // key result remains null
- }
- break;
- default:
- // both results will remain null
+ LogicalTypeAnnotation annotation = group.getLogicalTypeAnnotation();
+ if (annotation instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) {
+ Preconditions.checkArgument(
+ !group.isRepetition(Type.Repetition.REPEATED),
+ "Invalid list: top-level group is repeated: %s",
+ group);
+ Preconditions.checkArgument(
+ group.getFieldCount() == 1,
+ "Invalid list: does not contain single repeated field: %s",
+ group);
+
+ GroupType repeatedElement = group.getFields().get(0).asGroupType();
+ Preconditions.checkArgument(
+ repeatedElement.isRepetition(Type.Repetition.REPEATED),
+ "Invalid list: inner group is not repeated");
+ Preconditions.checkArgument(
+ repeatedElement.getFieldCount() <= 1,
+ "Invalid list: repeated group is not a single field: %s",
+ group);
+
+ Preconditions.checkArgument(
+ sType instanceof ArrayType, "Invalid list: %s is not an array", sType);
+ ArrayType array = (ArrayType) sType;
+ RowField element =
+ new RowField(
+ "element", array.getElementType(), "element of " + array.asSummaryString());
+
+ visitor.fieldNames.push(repeatedElement.getName());
+ try {
+ T elementResult = null;
+ if (repeatedElement.getFieldCount() > 0) {
+ elementResult = visitField(element, repeatedElement.getType(0), visitor);
+ }
+
+ return visitor.list(array, group, elementResult);
+
+ } finally {
+ visitor.fieldNames.pop();
+ }
+ } else if (annotation instanceof LogicalTypeAnnotation.MapLogicalTypeAnnotation) {
+ Preconditions.checkArgument(
+ !group.isRepetition(Type.Repetition.REPEATED),
+ "Invalid map: top-level group is repeated: %s",
+ group);
+ Preconditions.checkArgument(
+ group.getFieldCount() == 1,
+ "Invalid map: does not contain single repeated field: %s",
+ group);
+
+ GroupType repeatedKeyValue = group.getType(0).asGroupType();
+ Preconditions.checkArgument(
+ repeatedKeyValue.isRepetition(Type.Repetition.REPEATED),
+ "Invalid map: inner group is not repeated");
+ Preconditions.checkArgument(
+ repeatedKeyValue.getFieldCount() <= 2,
+ "Invalid map: repeated group does not have 2 fields");
+
+ Preconditions.checkArgument(
+ sType instanceof MapType, "Invalid map: %s is not a map", sType);
+ MapType map = (MapType) sType;
+ RowField keyField =
+ new RowField("key", map.getKeyType(), "key of " + map.asSummaryString());
+ RowField valueField =
+ new RowField("value", map.getValueType(), "value of " + map.asSummaryString());
+
+ visitor.fieldNames.push(repeatedKeyValue.getName());
+ try {
+ T keyResult = null;
+ T valueResult = null;
+ switch (repeatedKeyValue.getFieldCount()) {
+ case 2:
+ // if there are 2 fields, both key and value are projected
+ keyResult = visitField(keyField, repeatedKeyValue.getType(0), visitor);
+ valueResult = visitField(valueField, repeatedKeyValue.getType(1), visitor);
+ break;
+ case 1:
+ // if there is just one, use the name to determine what it is
+ Type keyOrValue = repeatedKeyValue.getType(0);
+ if (keyOrValue.getName().equalsIgnoreCase("key")) {
+ keyResult = visitField(keyField, keyOrValue, visitor);
+ // value result remains null
+ } else {
+ valueResult = visitField(valueField, keyOrValue, visitor);
+ // key result remains null
}
+ break;
+ default:
+ // both results will remain null
+ }
- return visitor.map(map, group, keyResult, valueResult);
-
- } finally {
- visitor.fieldNames.pop();
- }
+ return visitor.map(map, group, keyResult, valueResult);
- default:
+ } finally {
+ visitor.fieldNames.pop();
}
}
Preconditions.checkArgument(
diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetTypeVisitor.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetTypeVisitor.java
index 17518a175f..43c62a8c77 100644
--- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetTypeVisitor.java
+++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetTypeVisitor.java
@@ -23,8 +23,8 @@ import java.util.List;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
-import org.apache.parquet.schema.OriginalType;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.Type;
@@ -41,17 +41,11 @@ public class ParquetTypeVisitor<T> {
} else {
// if not a primitive, the typeId must be a group
GroupType group = type.asGroupType();
- OriginalType annotation = group.getOriginalType();
- if (annotation != null) {
- switch (annotation) {
- case LIST:
- return visitList(group, visitor);
-
- case MAP:
- return visitMap(group, visitor);
-
- default:
- }
+ LogicalTypeAnnotation annotation = group.getLogicalTypeAnnotation();
+ if (LogicalTypeAnnotation.listType().equals(annotation)) {
+ return visitList(group, visitor);
+ } else if (LogicalTypeAnnotation.mapType().equals(annotation)) {
+ return visitMap(group, visitor);
}
return visitor.struct(group, visitFields(group, visitor));
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java
index fcf05f542c..9c66cdf394 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java
@@ -753,7 +753,7 @@ public class SparkTableUtil {
Math.min(
partitions.size(), spark.sessionState().conf().parallelPartitionDiscoveryParallelism());
int numShufflePartitions = spark.sessionState().conf().numShufflePartitions();
- MetricsConfig metricsConfig = MetricsConfig.fromProperties(targetTable.properties());
+ MetricsConfig metricsConfig = MetricsConfig.forTable(targetTable);
String nameMappingString = targetTable.properties().get(TableProperties.DEFAULT_NAME_MAPPING);
NameMapping nameMapping =
nameMappingString != null ? NameMappingParser.fromJson(nameMappingString) : null;
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/ParquetWithSparkSchemaVisitor.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/ParquetWithSparkSchemaVisitor.java
index d74a76f94e..9480385d54 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/ParquetWithSparkSchemaVisitor.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/ParquetWithSparkSchemaVisitor.java
@@ -24,8 +24,8 @@ import org.apache.iceberg.avro.AvroSchemaUtil;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
-import org.apache.parquet.schema.OriginalType;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.Type;
import org.apache.parquet.schema.Type.Repetition;
@@ -59,106 +59,98 @@ public class ParquetWithSparkSchemaVisitor<T> {
} else {
// if not a primitive, the typeId must be a group
GroupType group = type.asGroupType();
- OriginalType annotation = group.getOriginalType();
- if (annotation != null) {
- switch (annotation) {
- case LIST:
- Preconditions.checkArgument(
- !group.isRepetition(Repetition.REPEATED),
- "Invalid list: top-level group is repeated: %s",
- group);
- Preconditions.checkArgument(
- group.getFieldCount() == 1,
- "Invalid list: does not contain single repeated field: %s",
- group);
-
- GroupType repeatedElement = group.getFields().get(0).asGroupType();
- Preconditions.checkArgument(
- repeatedElement.isRepetition(Repetition.REPEATED),
- "Invalid list: inner group is not repeated");
- Preconditions.checkArgument(
- repeatedElement.getFieldCount() <= 1,
- "Invalid list: repeated group is not a single field: %s",
- group);
-
- Preconditions.checkArgument(
- sType instanceof ArrayType, "Invalid list: %s is not an array", sType);
- ArrayType array = (ArrayType) sType;
- StructField element =
- new StructField(
- "element", array.elementType(), array.containsNull(), Metadata.empty());
-
- visitor.fieldNames.push(repeatedElement.getName());
- try {
- T elementResult = null;
- if (repeatedElement.getFieldCount() > 0) {
- elementResult = visitField(element, repeatedElement.getType(0), visitor);
- }
-
- return visitor.list(array, group, elementResult);
-
- } finally {
- visitor.fieldNames.pop();
- }
-
- case MAP:
- Preconditions.checkArgument(
- !group.isRepetition(Repetition.REPEATED),
- "Invalid map: top-level group is repeated: %s",
- group);
- Preconditions.checkArgument(
- group.getFieldCount() == 1,
- "Invalid map: does not contain single repeated field: %s",
- group);
-
- GroupType repeatedKeyValue = group.getType(0).asGroupType();
- Preconditions.checkArgument(
- repeatedKeyValue.isRepetition(Repetition.REPEATED),
- "Invalid map: inner group is not repeated");
- Preconditions.checkArgument(
- repeatedKeyValue.getFieldCount() <= 2,
- "Invalid map: repeated group does not have 2 fields");
-
- Preconditions.checkArgument(
- sType instanceof MapType, "Invalid map: %s is not a map", sType);
- MapType map = (MapType) sType;
- StructField keyField = new StructField("key", map.keyType(), false, Metadata.empty());
- StructField valueField =
- new StructField(
- "value", map.valueType(), map.valueContainsNull(), Metadata.empty());
-
- visitor.fieldNames.push(repeatedKeyValue.getName());
- try {
- T keyResult = null;
- T valueResult = null;
- switch (repeatedKeyValue.getFieldCount()) {
- case 2:
- // if there are 2 fields, both key and value are projected
- keyResult = visitField(keyField, repeatedKeyValue.getType(0), visitor);
- valueResult = visitField(valueField, repeatedKeyValue.getType(1), visitor);
- break;
- case 1:
- // if there is just one, use the name to determine what it is
- Type keyOrValue = repeatedKeyValue.getType(0);
- if (keyOrValue.getName().equalsIgnoreCase("key")) {
- keyResult = visitField(keyField, keyOrValue, visitor);
- // value result remains null
- } else {
- valueResult = visitField(valueField, keyOrValue, visitor);
- // key result remains null
- }
- break;
- default:
- // both results will remain null
+ LogicalTypeAnnotation annotation = group.getLogicalTypeAnnotation();
+ if (LogicalTypeAnnotation.listType().equals(annotation)) {
+ Preconditions.checkArgument(
+ !group.isRepetition(Repetition.REPEATED),
+ "Invalid list: top-level group is repeated: %s",
+ group);
+ Preconditions.checkArgument(
+ group.getFieldCount() == 1,
+ "Invalid list: does not contain single repeated field: %s",
+ group);
+
+ GroupType repeatedElement = group.getFields().get(0).asGroupType();
+ Preconditions.checkArgument(
+ repeatedElement.isRepetition(Repetition.REPEATED),
+ "Invalid list: inner group is not repeated");
+ Preconditions.checkArgument(
+ repeatedElement.getFieldCount() <= 1,
+ "Invalid list: repeated group is not a single field: %s",
+ group);
+
+ Preconditions.checkArgument(
+ sType instanceof ArrayType, "Invalid list: %s is not an array", sType);
+ ArrayType array = (ArrayType) sType;
+ StructField element =
+ new StructField("element", array.elementType(), array.containsNull(), Metadata.empty());
+
+ visitor.fieldNames.push(repeatedElement.getName());
+ try {
+ T elementResult = null;
+ if (repeatedElement.getFieldCount() > 0) {
+ elementResult = visitField(element, repeatedElement.getType(0), visitor);
+ }
+
+ return visitor.list(array, group, elementResult);
+
+ } finally {
+ visitor.fieldNames.pop();
+ }
+ } else if (LogicalTypeAnnotation.mapType().equals(annotation)) {
+ Preconditions.checkArgument(
+ !group.isRepetition(Repetition.REPEATED),
+ "Invalid map: top-level group is repeated: %s",
+ group);
+ Preconditions.checkArgument(
+ group.getFieldCount() == 1,
+ "Invalid map: does not contain single repeated field: %s",
+ group);
+
+ GroupType repeatedKeyValue = group.getType(0).asGroupType();
+ Preconditions.checkArgument(
+ repeatedKeyValue.isRepetition(Repetition.REPEATED),
+ "Invalid map: inner group is not repeated");
+ Preconditions.checkArgument(
+ repeatedKeyValue.getFieldCount() <= 2,
+ "Invalid map: repeated group does not have 2 fields");
+
+ Preconditions.checkArgument(
+ sType instanceof MapType, "Invalid map: %s is not a map", sType);
+ MapType map = (MapType) sType;
+ StructField keyField = new StructField("key", map.keyType(), false, Metadata.empty());
+ StructField valueField =
+ new StructField("value", map.valueType(), map.valueContainsNull(), Metadata.empty());
+
+ visitor.fieldNames.push(repeatedKeyValue.getName());
+ try {
+ T keyResult = null;
+ T valueResult = null;
+ switch (repeatedKeyValue.getFieldCount()) {
+ case 2:
+ // if there are 2 fields, both key and value are projected
+ keyResult = visitField(keyField, repeatedKeyValue.getType(0), visitor);
+ valueResult = visitField(valueField, repeatedKeyValue.getType(1), visitor);
+ break;
+ case 1:
+ // if there is just one, use the name to determine what it is
+ Type keyOrValue = repeatedKeyValue.getType(0);
+ if (keyOrValue.getName().equalsIgnoreCase("key")) {
+ keyResult = visitField(keyField, keyOrValue, visitor);
+ // value result remains null
+ } else {
+ valueResult = visitField(valueField, keyOrValue, visitor);
+ // key result remains null
}
+ break;
+ default:
+ // both results will remain null
+ }
- return visitor.map(map, group, keyResult, valueResult);
-
- } finally {
- visitor.fieldNames.pop();
- }
+ return visitor.map(map, group, keyResult, valueResult);
- default:
+ } finally {
+ visitor.fieldNames.pop();
}
}
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkCatalogOperations.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkCatalogOperations.java
index 186042283b..5b460a0b52 100644
--- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkCatalogOperations.java
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkCatalogOperations.java
@@ -28,11 +28,11 @@ import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.types.Types;
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+import org.apache.spark.sql.connector.catalog.Column;
import org.apache.spark.sql.connector.catalog.Identifier;
import org.apache.spark.sql.connector.catalog.Table;
import org.apache.spark.sql.connector.catalog.TableChange;
import org.apache.spark.sql.types.DataTypes;
-import org.apache.spark.sql.types.StructField;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestTemplate;
@@ -107,8 +107,8 @@ public class TestSparkCatalogOperations extends CatalogTestBase {
assertThat(table).as("Should return updated table").isNotNull();
- StructField expectedField = DataTypes.createStructField(fieldName, DataTypes.StringType, true);
- assertThat(table.schema().fields()[2])
+ Column expectedField = Column.create(fieldName, DataTypes.StringType, true);
+ assertThat(table.columns()[2])
.as("Adding a column to a table should return the updated table with the new column")
.isEqualTo(expectedField);
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
index f411920a5d..6b7d861364 100644
--- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
@@ -35,7 +35,6 @@ import static org.apache.iceberg.TableProperties.PARQUET_COMPRESSION;
import static org.apache.iceberg.spark.SparkSQLProperties.COMPRESSION_CODEC;
import static org.apache.iceberg.spark.SparkSQLProperties.COMPRESSION_LEVEL;
import static org.apache.iceberg.spark.SparkSQLProperties.COMPRESSION_STRATEGY;
-import static org.apache.parquet.format.converter.ParquetMetadataConverter.NO_FILTER;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.List;
@@ -72,6 +71,7 @@ import org.apache.orc.OrcFile;
import org.apache.orc.Reader;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+import org.apache.parquet.hadoop.util.HadoopInputFile;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
@@ -251,7 +251,8 @@ public class TestCompressionSettings extends CatalogTestBase {
return orcReader.getCompressionKind().name();
case PARQUET:
ParquetMetadata footer =
- ParquetFileReader.readFooter(CONF, new Path(inputFile.location()), NO_FILTER);
+ ParquetFileReader.open(HadoopInputFile.fromPath(new Path(inputFile.location()), CONF))
+ .getFooter();
return footer.getBlocks().get(0).getColumns().get(0).getCodec().name();
default:
FileContext fc = FileContext.getFileContext(CONF);