You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by et...@apache.org on 2024/10/16 12:32:58 UTC
(iceberg) branch main updated: Build, Spark, Flink: Bump junit from 5.10.1 to 5.11.1 (#11262)
This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 5e279c868f Build, Spark, Flink: Bump junit from 5.10.1 to 5.11.1 (#11262)
5e279c868f is described below
commit 5e279c868f8e2087f88ae779d8cc8474768bc5e2
Author: Tom Tanaka <43...@users.noreply.github.com>
AuthorDate: Wed Oct 16 21:32:51 2024 +0900
Build, Spark, Flink: Bump junit from 5.10.1 to 5.11.1 (#11262)
---
.../apache/iceberg/ParameterizedTestExtension.java | 11 ++++++-----
.../flink/sink/TestFlinkIcebergSinkV2Branch.java | 17 ++++++++++++----
.../iceberg/flink/source/TestFlinkSourceSql.java | 2 ++
.../iceberg/flink/source/TestIcebergSourceSql.java | 2 ++
.../flink/sink/TestFlinkIcebergSinkV2Branch.java | 17 ++++++++++++----
.../flink/sink/TestIcebergSinkV2Branch.java | 11 +----------
.../iceberg/flink/source/TestFlinkSourceSql.java | 2 ++
.../iceberg/flink/source/TestIcebergSourceSql.java | 2 ++
.../flink/sink/TestFlinkIcebergSinkV2Branch.java | 17 ++++++++++++----
.../flink/sink/TestIcebergSinkV2Branch.java | 11 +----------
.../iceberg/flink/source/TestFlinkSourceSql.java | 2 ++
.../iceberg/flink/source/TestIcebergSourceSql.java | 2 ++
gradle/libs.versions.toml | 2 +-
.../iceberg/mr/TestInputFormatReaderDeletes.java | 23 ++++++++++------------
.../spark/source/TestSparkReaderDeletes.java | 19 +++++++++---------
.../spark/source/TestSparkReaderDeletes.java | 19 +++++++++---------
.../spark/extensions/ExtensionsTestBase.java | 2 ++
.../spark/source/TestSparkReaderDeletes.java | 19 +++++++++---------
.../iceberg/spark/sql/TestAggregatePushDown.java | 2 ++
.../spark/sql/TestUnpartitionedWritesToBranch.java | 2 ++
20 files changed, 103 insertions(+), 81 deletions(-)
diff --git a/api/src/test/java/org/apache/iceberg/ParameterizedTestExtension.java b/api/src/test/java/org/apache/iceberg/ParameterizedTestExtension.java
index 59652bab98..2f625092ff 100644
--- a/api/src/test/java/org/apache/iceberg/ParameterizedTestExtension.java
+++ b/api/src/test/java/org/apache/iceberg/ParameterizedTestExtension.java
@@ -73,13 +73,10 @@ public class ParameterizedTestExtension implements TestTemplateInvocationContext
// Search method annotated with @Parameters
final List<Method> parameterProviders =
AnnotationSupport.findAnnotatedMethods(
- context.getRequiredTestClass(), Parameters.class, HierarchyTraversalMode.TOP_DOWN);
+ context.getRequiredTestClass(), Parameters.class, HierarchyTraversalMode.BOTTOM_UP);
if (parameterProviders.isEmpty()) {
throw new IllegalStateException("Cannot find any parameter provider");
}
- if (parameterProviders.size() > 1) {
- throw new IllegalStateException("Multiple parameter providers are found");
- }
Method parameterProvider = parameterProviders.get(0);
// Get potential test name
@@ -225,7 +222,11 @@ public class ParameterizedTestExtension implements TestTemplateInvocationContext
Stream<Object[]> parameterValueStream, String testNameTemplate, ExtensionContext context) {
// Search fields annotated by @Parameter
final List<Field> parameterFields =
- AnnotationSupport.findAnnotatedFields(context.getRequiredTestClass(), Parameter.class);
+ AnnotationSupport.findAnnotatedFields(
+ context.getRequiredTestClass(),
+ Parameter.class,
+ field -> true,
+ HierarchyTraversalMode.BOTTOM_UP);
// Use constructor parameter style
if (parameterFields.isEmpty()) {
diff --git a/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/sink/TestFlinkIcebergSinkV2Branch.java b/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/sink/TestFlinkIcebergSinkV2Branch.java
index 0b0c55f51c..52826305b4 100644
--- a/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/sink/TestFlinkIcebergSinkV2Branch.java
+++ b/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/sink/TestFlinkIcebergSinkV2Branch.java
@@ -46,12 +46,21 @@ public class TestFlinkIcebergSinkV2Branch extends TestFlinkIcebergSinkV2Base {
private static final HadoopCatalogExtension CATALOG_EXTENSION =
new HadoopCatalogExtension(DATABASE, TestFixtures.TABLE);
- @Parameter(index = 0)
- private String branch;
+ @Parameter(index = 4)
+ protected String branch;
- @Parameters(name = "branch = {0}")
+ @Parameters(
+ name =
+ "FileFormat={0}, Parallelism={1}, Partitioned={2}, WriteDistributionMode={3}, Branch={4}")
public static Object[][] parameters() {
- return new Object[][] {new Object[] {"main"}, new Object[] {"testBranch"}};
+ return new Object[][] {
+ new Object[] {
+ FileFormat.AVRO, 1, false, TableProperties.WRITE_DISTRIBUTION_MODE_NONE, "main"
+ },
+ new Object[] {
+ FileFormat.AVRO, 1, false, TableProperties.WRITE_DISTRIBUTION_MODE_NONE, "testBranch"
+ }
+ };
}
@BeforeEach
diff --git a/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/source/TestFlinkSourceSql.java b/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/source/TestFlinkSourceSql.java
index e1162c3225..2dc5bc5c65 100644
--- a/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/source/TestFlinkSourceSql.java
+++ b/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/source/TestFlinkSourceSql.java
@@ -33,10 +33,12 @@ import org.apache.iceberg.data.Record;
import org.apache.iceberg.flink.TestFixtures;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/** Use the FlinkSource */
public class TestFlinkSourceSql extends TestSqlBase {
+ @BeforeEach
@Override
public void before() throws IOException {
SqlHelpers.sql(
diff --git a/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java b/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java
index 548940a842..66bdeee1d4 100644
--- a/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java
+++ b/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java
@@ -40,6 +40,7 @@ import org.apache.iceberg.flink.TestHelpers;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/** Use the IcebergSource (FLIP-27) */
@@ -49,6 +50,7 @@ public class TestIcebergSourceSql extends TestSqlBase {
required(1, "t1", Types.TimestampType.withoutZone()),
required(2, "t2", Types.LongType.get()));
+ @BeforeEach
@Override
public void before() throws IOException {
TableEnvironment tableEnvironment = getTableEnv();
diff --git a/flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/sink/TestFlinkIcebergSinkV2Branch.java b/flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/sink/TestFlinkIcebergSinkV2Branch.java
index a2e9be9303..56cba8f460 100644
--- a/flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/sink/TestFlinkIcebergSinkV2Branch.java
+++ b/flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/sink/TestFlinkIcebergSinkV2Branch.java
@@ -46,12 +46,21 @@ public class TestFlinkIcebergSinkV2Branch extends TestFlinkIcebergSinkV2Base {
static final HadoopCatalogExtension CATALOG_EXTENSION =
new HadoopCatalogExtension(DATABASE, TestFixtures.TABLE);
- @Parameter(index = 0)
- private String branch;
+ @Parameter(index = 4)
+ protected String branch;
- @Parameters(name = "branch = {0}")
+ @Parameters(
+ name =
+ "FileFormat={0}, Parallelism={1}, Partitioned={2}, WriteDistributionMode={3}, Branch={4}")
public static Object[][] parameters() {
- return new Object[][] {new Object[] {"main"}, new Object[] {"testBranch"}};
+ return new Object[][] {
+ new Object[] {
+ FileFormat.AVRO, 1, false, TableProperties.WRITE_DISTRIBUTION_MODE_NONE, "main"
+ },
+ new Object[] {
+ FileFormat.AVRO, 1, false, TableProperties.WRITE_DISTRIBUTION_MODE_NONE, "testBranch"
+ }
+ };
}
@BeforeEach
diff --git a/flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkV2Branch.java b/flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkV2Branch.java
index 93da40ab5c..4896f7f48c 100644
--- a/flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkV2Branch.java
+++ b/flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkV2Branch.java
@@ -23,9 +23,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import java.io.IOException;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.iceberg.FileFormat;
-import org.apache.iceberg.Parameter;
import org.apache.iceberg.ParameterizedTestExtension;
-import org.apache.iceberg.Parameters;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.SnapshotRef;
import org.apache.iceberg.TableProperties;
@@ -40,15 +38,8 @@ import org.junit.jupiter.api.extension.ExtendWith;
@ExtendWith(ParameterizedTestExtension.class)
public class TestIcebergSinkV2Branch extends TestFlinkIcebergSinkV2Branch {
- @Parameter(index = 0)
- private String branch;
-
- @Parameters(name = "branch = {0}")
- public static Object[][] parameters() {
- return new Object[][] {new Object[] {"main"}, new Object[] {"testBranch"}};
- }
-
@BeforeEach
+ @Override
public void before() throws IOException {
table =
CATALOG_EXTENSION
diff --git a/flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/source/TestFlinkSourceSql.java b/flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/source/TestFlinkSourceSql.java
index e1162c3225..2dc5bc5c65 100644
--- a/flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/source/TestFlinkSourceSql.java
+++ b/flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/source/TestFlinkSourceSql.java
@@ -33,10 +33,12 @@ import org.apache.iceberg.data.Record;
import org.apache.iceberg.flink.TestFixtures;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/** Use the FlinkSource */
public class TestFlinkSourceSql extends TestSqlBase {
+ @BeforeEach
@Override
public void before() throws IOException {
SqlHelpers.sql(
diff --git a/flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java b/flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java
index 548940a842..66bdeee1d4 100644
--- a/flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java
+++ b/flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java
@@ -40,6 +40,7 @@ import org.apache.iceberg.flink.TestHelpers;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/** Use the IcebergSource (FLIP-27) */
@@ -49,6 +50,7 @@ public class TestIcebergSourceSql extends TestSqlBase {
required(1, "t1", Types.TimestampType.withoutZone()),
required(2, "t2", Types.LongType.get()));
+ @BeforeEach
@Override
public void before() throws IOException {
TableEnvironment tableEnvironment = getTableEnv();
diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/sink/TestFlinkIcebergSinkV2Branch.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/sink/TestFlinkIcebergSinkV2Branch.java
index a2e9be9303..56cba8f460 100644
--- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/sink/TestFlinkIcebergSinkV2Branch.java
+++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/sink/TestFlinkIcebergSinkV2Branch.java
@@ -46,12 +46,21 @@ public class TestFlinkIcebergSinkV2Branch extends TestFlinkIcebergSinkV2Base {
static final HadoopCatalogExtension CATALOG_EXTENSION =
new HadoopCatalogExtension(DATABASE, TestFixtures.TABLE);
- @Parameter(index = 0)
- private String branch;
+ @Parameter(index = 4)
+ protected String branch;
- @Parameters(name = "branch = {0}")
+ @Parameters(
+ name =
+ "FileFormat={0}, Parallelism={1}, Partitioned={2}, WriteDistributionMode={3}, Branch={4}")
public static Object[][] parameters() {
- return new Object[][] {new Object[] {"main"}, new Object[] {"testBranch"}};
+ return new Object[][] {
+ new Object[] {
+ FileFormat.AVRO, 1, false, TableProperties.WRITE_DISTRIBUTION_MODE_NONE, "main"
+ },
+ new Object[] {
+ FileFormat.AVRO, 1, false, TableProperties.WRITE_DISTRIBUTION_MODE_NONE, "testBranch"
+ }
+ };
}
@BeforeEach
diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkV2Branch.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkV2Branch.java
index 93da40ab5c..4896f7f48c 100644
--- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkV2Branch.java
+++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkV2Branch.java
@@ -23,9 +23,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import java.io.IOException;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.iceberg.FileFormat;
-import org.apache.iceberg.Parameter;
import org.apache.iceberg.ParameterizedTestExtension;
-import org.apache.iceberg.Parameters;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.SnapshotRef;
import org.apache.iceberg.TableProperties;
@@ -40,15 +38,8 @@ import org.junit.jupiter.api.extension.ExtendWith;
@ExtendWith(ParameterizedTestExtension.class)
public class TestIcebergSinkV2Branch extends TestFlinkIcebergSinkV2Branch {
- @Parameter(index = 0)
- private String branch;
-
- @Parameters(name = "branch = {0}")
- public static Object[][] parameters() {
- return new Object[][] {new Object[] {"main"}, new Object[] {"testBranch"}};
- }
-
@BeforeEach
+ @Override
public void before() throws IOException {
table =
CATALOG_EXTENSION
diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/TestFlinkSourceSql.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/TestFlinkSourceSql.java
index e1162c3225..2dc5bc5c65 100644
--- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/TestFlinkSourceSql.java
+++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/TestFlinkSourceSql.java
@@ -33,10 +33,12 @@ import org.apache.iceberg.data.Record;
import org.apache.iceberg.flink.TestFixtures;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/** Use the FlinkSource */
public class TestFlinkSourceSql extends TestSqlBase {
+ @BeforeEach
@Override
public void before() throws IOException {
SqlHelpers.sql(
diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java
index 548940a842..66bdeee1d4 100644
--- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java
+++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java
@@ -40,6 +40,7 @@ import org.apache.iceberg.flink.TestHelpers;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/** Use the IcebergSource (FLIP-27) */
@@ -49,6 +50,7 @@ public class TestIcebergSourceSql extends TestSqlBase {
required(1, "t1", Types.TimestampType.withoutZone()),
required(2, "t2", Types.LongType.get()));
+ @BeforeEach
@Override
public void before() throws IOException {
TableEnvironment tableEnvironment = getTableEnv();
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index d49c082b09..34fd2ad8ba 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -63,7 +63,7 @@ jakarta-servlet-api = "6.1.0"
jaxb-api = "2.3.1"
jaxb-runtime = "2.3.9"
jetty = "11.0.24"
-junit = "5.10.1"
+junit = "5.11.1"
junit-platform = "1.11.2"
kafka = "3.8.0"
kryo-shaded = "4.0.3"
diff --git a/mr/src/test/java/org/apache/iceberg/mr/TestInputFormatReaderDeletes.java b/mr/src/test/java/org/apache/iceberg/mr/TestInputFormatReaderDeletes.java
index a5f1089692..2cb41f1129 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/TestInputFormatReaderDeletes.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/TestInputFormatReaderDeletes.java
@@ -49,21 +49,18 @@ public class TestInputFormatReaderDeletes extends DeleteReadTests {
private final HadoopTables tables = new HadoopTables(conf);
private TestHelper helper;
- // parametrized variables
- @Parameter private String inputFormat;
-
@Parameter(index = 1)
- private FileFormat fileFormat;
+ private String inputFormat;
- @Parameters(name = "inputFormat = {0}, fileFormat = {1}")
+ @Parameters(name = "fileFormat = {0}, inputFormat = {1}")
public static Object[][] parameters() {
return new Object[][] {
- {"IcebergInputFormat", FileFormat.PARQUET},
- {"IcebergInputFormat", FileFormat.AVRO},
- {"IcebergInputFormat", FileFormat.ORC},
- {"MapredIcebergInputFormat", FileFormat.PARQUET},
- {"MapredIcebergInputFormat", FileFormat.AVRO},
- {"MapredIcebergInputFormat", FileFormat.ORC},
+ {FileFormat.PARQUET, "IcebergInputFormat"},
+ {FileFormat.AVRO, "IcebergInputFormat"},
+ {FileFormat.ORC, "IcebergInputFormat"},
+ {FileFormat.PARQUET, "MapredIcebergInputFormat"},
+ {FileFormat.AVRO, "MapredIcebergInputFormat"},
+ {FileFormat.ORC, "MapredIcebergInputFormat"},
};
}
@@ -78,9 +75,9 @@ public class TestInputFormatReaderDeletes extends DeleteReadTests {
protected Table createTable(String name, Schema schema, PartitionSpec spec) throws IOException {
Table table;
- File location = temp.resolve(inputFormat).resolve(fileFormat.name()).toFile();
+ File location = temp.resolve(inputFormat).resolve(format.name()).toFile();
assertThat(location.mkdirs()).isTrue();
- helper = new TestHelper(conf, tables, location.toString(), schema, spec, fileFormat, temp);
+ helper = new TestHelper(conf, tables, location.toString(), schema, spec, format, temp);
table = helper.createTable();
TableOperations ops = ((BaseTable) table).operations();
diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java
index 76a4143fcb..bde87778ad 100644
--- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java
+++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java
@@ -37,6 +37,7 @@ import org.apache.iceberg.CombinedScanTask;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.DataFiles;
import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
import org.apache.iceberg.Files;
import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.Parameter;
@@ -98,18 +99,16 @@ public class TestSparkReaderDeletes extends DeleteReadTests {
protected static SparkSession spark = null;
protected static HiveCatalog catalog = null;
- @Parameter private String format;
-
@Parameter(index = 1)
private boolean vectorized;
@Parameters(name = "format = {0}, vectorized = {1}")
public static Object[][] parameters() {
return new Object[][] {
- new Object[] {"parquet", false},
- new Object[] {"parquet", true},
- new Object[] {"orc", false},
- new Object[] {"avro", false}
+ new Object[] {FileFormat.PARQUET, false},
+ new Object[] {FileFormat.PARQUET, true},
+ new Object[] {FileFormat.ORC, false},
+ new Object[] {FileFormat.AVRO, false}
};
}
@@ -163,14 +162,14 @@ public class TestSparkReaderDeletes extends DeleteReadTests {
TableOperations ops = ((BaseTable) table).operations();
TableMetadata meta = ops.current();
ops.commit(meta, meta.upgradeToFormatVersion(2));
- table.updateProperties().set(TableProperties.DEFAULT_FILE_FORMAT, format).commit();
- if (format.equals("parquet") || format.equals("orc")) {
+ table.updateProperties().set(TableProperties.DEFAULT_FILE_FORMAT, format.name()).commit();
+ if (format.equals(FileFormat.PARQUET) || format.equals(FileFormat.ORC)) {
String vectorizationEnabled =
- format.equals("parquet")
+ format.equals(FileFormat.PARQUET)
? TableProperties.PARQUET_VECTORIZATION_ENABLED
: TableProperties.ORC_VECTORIZATION_ENABLED;
String batchSize =
- format.equals("parquet")
+ format.equals(FileFormat.PARQUET)
? TableProperties.PARQUET_BATCH_SIZE
: TableProperties.ORC_BATCH_SIZE;
table.updateProperties().set(vectorizationEnabled, String.valueOf(vectorized)).commit();
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java
index 4643836542..29c2d4b39a 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java
@@ -37,6 +37,7 @@ import org.apache.iceberg.CombinedScanTask;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.DataFiles;
import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
import org.apache.iceberg.Files;
import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.Parameter;
@@ -99,8 +100,6 @@ public class TestSparkReaderDeletes extends DeleteReadTests {
protected static SparkSession spark = null;
protected static HiveCatalog catalog = null;
- @Parameter private String format;
-
@Parameter(index = 1)
private boolean vectorized;
@@ -110,10 +109,10 @@ public class TestSparkReaderDeletes extends DeleteReadTests {
@Parameters(name = "format = {0}, vectorized = {1}, planningMode = {2}")
public static Object[][] parameters() {
return new Object[][] {
- new Object[] {"parquet", false, PlanningMode.DISTRIBUTED},
- new Object[] {"parquet", true, PlanningMode.LOCAL},
- new Object[] {"orc", false, PlanningMode.DISTRIBUTED},
- new Object[] {"avro", false, PlanningMode.LOCAL}
+ new Object[] {FileFormat.PARQUET, false, PlanningMode.DISTRIBUTED},
+ new Object[] {FileFormat.PARQUET, true, PlanningMode.LOCAL},
+ new Object[] {FileFormat.ORC, false, PlanningMode.DISTRIBUTED},
+ new Object[] {FileFormat.AVRO, false, PlanningMode.LOCAL}
};
}
@@ -169,17 +168,17 @@ public class TestSparkReaderDeletes extends DeleteReadTests {
ops.commit(meta, meta.upgradeToFormatVersion(2));
table
.updateProperties()
- .set(TableProperties.DEFAULT_FILE_FORMAT, format)
+ .set(TableProperties.DEFAULT_FILE_FORMAT, format.name())
.set(TableProperties.DATA_PLANNING_MODE, planningMode.modeName())
.set(TableProperties.DELETE_PLANNING_MODE, planningMode.modeName())
.commit();
- if (format.equals("parquet") || format.equals("orc")) {
+ if (format.equals(FileFormat.PARQUET) || format.equals(FileFormat.ORC)) {
String vectorizationEnabled =
- format.equals("parquet")
+ format.equals(FileFormat.PARQUET)
? TableProperties.PARQUET_VECTORIZATION_ENABLED
: TableProperties.ORC_VECTORIZATION_ENABLED;
String batchSize =
- format.equals("parquet")
+ format.equals(FileFormat.PARQUET)
? TableProperties.PARQUET_BATCH_SIZE
: TableProperties.ORC_BATCH_SIZE;
table.updateProperties().set(vectorizationEnabled, String.valueOf(vectorized)).commit();
diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java
index 005d5e29d5..578845e3da 100644
--- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java
+++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java
@@ -43,6 +43,8 @@ public abstract class ExtensionsTestBase extends CatalogTestBase {
metastore.start();
TestBase.hiveConf = metastore.hiveConf();
+ TestBase.spark.close();
+
TestBase.spark =
SparkSession.builder()
.master("local[2]")
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java
index 4643836542..29c2d4b39a 100644
--- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java
@@ -37,6 +37,7 @@ import org.apache.iceberg.CombinedScanTask;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.DataFiles;
import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
import org.apache.iceberg.Files;
import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.Parameter;
@@ -99,8 +100,6 @@ public class TestSparkReaderDeletes extends DeleteReadTests {
protected static SparkSession spark = null;
protected static HiveCatalog catalog = null;
- @Parameter private String format;
-
@Parameter(index = 1)
private boolean vectorized;
@@ -110,10 +109,10 @@ public class TestSparkReaderDeletes extends DeleteReadTests {
@Parameters(name = "format = {0}, vectorized = {1}, planningMode = {2}")
public static Object[][] parameters() {
return new Object[][] {
- new Object[] {"parquet", false, PlanningMode.DISTRIBUTED},
- new Object[] {"parquet", true, PlanningMode.LOCAL},
- new Object[] {"orc", false, PlanningMode.DISTRIBUTED},
- new Object[] {"avro", false, PlanningMode.LOCAL}
+ new Object[] {FileFormat.PARQUET, false, PlanningMode.DISTRIBUTED},
+ new Object[] {FileFormat.PARQUET, true, PlanningMode.LOCAL},
+ new Object[] {FileFormat.ORC, false, PlanningMode.DISTRIBUTED},
+ new Object[] {FileFormat.AVRO, false, PlanningMode.LOCAL}
};
}
@@ -169,17 +168,17 @@ public class TestSparkReaderDeletes extends DeleteReadTests {
ops.commit(meta, meta.upgradeToFormatVersion(2));
table
.updateProperties()
- .set(TableProperties.DEFAULT_FILE_FORMAT, format)
+ .set(TableProperties.DEFAULT_FILE_FORMAT, format.name())
.set(TableProperties.DATA_PLANNING_MODE, planningMode.modeName())
.set(TableProperties.DELETE_PLANNING_MODE, planningMode.modeName())
.commit();
- if (format.equals("parquet") || format.equals("orc")) {
+ if (format.equals(FileFormat.PARQUET) || format.equals(FileFormat.ORC)) {
String vectorizationEnabled =
- format.equals("parquet")
+ format.equals(FileFormat.PARQUET)
? TableProperties.PARQUET_VECTORIZATION_ENABLED
: TableProperties.ORC_VECTORIZATION_ENABLED;
String batchSize =
- format.equals("parquet")
+ format.equals(FileFormat.PARQUET)
? TableProperties.PARQUET_BATCH_SIZE
: TableProperties.ORC_BATCH_SIZE;
table.updateProperties().set(vectorizationEnabled, String.valueOf(vectorized)).commit();
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java
index 7e9bdeec8a..6e09252704 100644
--- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java
@@ -54,6 +54,8 @@ public class TestAggregatePushDown extends CatalogTestBase {
metastore.start();
TestBase.hiveConf = metastore.hiveConf();
+ TestBase.spark.close();
+
TestBase.spark =
SparkSession.builder()
.master("local[2]")
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestUnpartitionedWritesToBranch.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestUnpartitionedWritesToBranch.java
index a0c664b03b..3df5e9cdf5 100644
--- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestUnpartitionedWritesToBranch.java
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestUnpartitionedWritesToBranch.java
@@ -22,6 +22,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
import org.apache.iceberg.Table;
import org.apache.iceberg.exceptions.ValidationException;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestTemplate;
public class TestUnpartitionedWritesToBranch extends UnpartitionedWritesTestBase {
@@ -29,6 +30,7 @@ public class TestUnpartitionedWritesToBranch extends UnpartitionedWritesTestBase
private static final String BRANCH = "test";
@Override
+ @BeforeEach
public void createTables() {
super.createTables();
Table table = validationCatalog.loadTable(tableIdent);