You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by at...@apache.org on 2026/06/16 11:51:45 UTC
(iceberg) branch main updated: Spark: Reduce TestSparkDataFile row count to speed up partitioned cases (#16821)
This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 69063fb162 Spark: Reduce TestSparkDataFile row count to speed up partitioned cases (#16821)
69063fb162 is described below
commit 69063fb162e835c3dc1ff7bbcf5526029771c6d0
Author: Sebastian Baunsgaard <ba...@apache.org>
AuthorDate: Tue Jun 16 13:51:39 2026 +0200
Spark: Reduce TestSparkDataFile row count to speed up partitioned cases (#16821)
TestSparkDataFile writes random rows through a 14-field partition spec, so
nearly every row lands in its own partition and produces a separate data file
(plus a position-delete file per data file). The cost scales with the row
count, not with coverage: every generated row already exercises all column
types and all 14 partition transforms, and both delete-file types still
round-trip.
Reduce the generated row count from 200 to 40 in the v3.5/v4.0/v4.1 test
copies. The suite drops ~58% (27.4s -> 11.5s locally on Spark 4.1) with
identical pass/skip counts.
---
.../test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java | 2 +-
.../test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java | 2 +-
.../test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
index 2122f2579e..fdf43bb075 100644
--- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
@@ -169,7 +169,7 @@ public class TestSparkDataFile {
}
private void checkSparkContentFiles(Table table) throws IOException {
- Iterable<InternalRow> rows = RandomData.generateSpark(table.schema(), 200, 0);
+ Iterable<InternalRow> rows = RandomData.generateSpark(table.schema(), 40, 0);
JavaRDD<InternalRow> rdd = sparkContext.parallelize(Lists.newArrayList(rows));
Dataset<Row> df =
spark.internalCreateDataFrame(
diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
index d719ca6751..8b55cc0fe6 100644
--- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
+++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
@@ -170,7 +170,7 @@ public class TestSparkDataFile {
}
private void checkSparkContentFiles(Table table) throws IOException {
- Iterable<InternalRow> rows = RandomData.generateSpark(table.schema(), 200, 0);
+ Iterable<InternalRow> rows = RandomData.generateSpark(table.schema(), 40, 0);
JavaRDD<InternalRow> rdd = sparkContext.parallelize(Lists.newArrayList(rows));
Preconditions.checkArgument(
spark instanceof org.apache.spark.sql.classic.SparkSession,
diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
index d719ca6751..8b55cc0fe6 100644
--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
+++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
@@ -170,7 +170,7 @@ public class TestSparkDataFile {
}
private void checkSparkContentFiles(Table table) throws IOException {
- Iterable<InternalRow> rows = RandomData.generateSpark(table.schema(), 200, 0);
+ Iterable<InternalRow> rows = RandomData.generateSpark(table.schema(), 40, 0);
JavaRDD<InternalRow> rdd = sparkContext.parallelize(Lists.newArrayList(rows));
Preconditions.checkArgument(
spark instanceof org.apache.spark.sql.classic.SparkSession,