You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by si...@apache.org on 2026/05/01 14:50:23 UTC
(iceberg) branch main updated: Spark 3.4, 3.5, 4.0: Migrate SparkWriteBuilder to SupportsOverwriteV2 (#16178)
This is an automated email from the ASF dual-hosted git repository.
singhpk234 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 53f1f1a0a7 Spark 3.4, 3.5, 4.0: Migrate SparkWriteBuilder to SupportsOverwriteV2 (#16178)
53f1f1a0a7 is described below
commit 53f1f1a0a7fbdb013676358e8198f2cbd88ef296
Author: drexler-sky <ev...@gmail.com>
AuthorDate: Fri May 1 07:50:18 2026 -0700
Spark 3.4, 3.5, 4.0: Migrate SparkWriteBuilder to SupportsOverwriteV2 (#16178)
---
.../org/apache/iceberg/spark/source/SparkWriteBuilder.java | 12 ++++++------
.../org/apache/iceberg/spark/source/SparkWriteBuilder.java | 12 ++++++------
.../org/apache/iceberg/spark/source/SparkWriteBuilder.java | 12 ++++++------
3 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java
index e608a40b72..df4566da0c 100644
--- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java
+++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java
@@ -27,27 +27,27 @@ import org.apache.iceberg.UpdateSchema;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.spark.SparkFilters;
import org.apache.iceberg.spark.SparkSchemaUtil;
import org.apache.iceberg.spark.SparkUtil;
+import org.apache.iceberg.spark.SparkV2Filters;
import org.apache.iceberg.spark.SparkWriteConf;
import org.apache.iceberg.spark.SparkWriteRequirements;
import org.apache.iceberg.types.TypeUtil;
import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.connector.expressions.filter.Predicate;
import org.apache.spark.sql.connector.read.Scan;
import org.apache.spark.sql.connector.write.BatchWrite;
import org.apache.spark.sql.connector.write.LogicalWriteInfo;
import org.apache.spark.sql.connector.write.RowLevelOperation.Command;
import org.apache.spark.sql.connector.write.SupportsDynamicOverwrite;
-import org.apache.spark.sql.connector.write.SupportsOverwrite;
+import org.apache.spark.sql.connector.write.SupportsOverwriteV2;
import org.apache.spark.sql.connector.write.Write;
import org.apache.spark.sql.connector.write.WriteBuilder;
import org.apache.spark.sql.connector.write.streaming.StreamingWrite;
-import org.apache.spark.sql.sources.Filter;
import org.apache.spark.sql.types.LongType$;
import org.apache.spark.sql.types.StructType;
-class SparkWriteBuilder implements WriteBuilder, SupportsDynamicOverwrite, SupportsOverwrite {
+class SparkWriteBuilder implements WriteBuilder, SupportsDynamicOverwrite, SupportsOverwriteV2 {
private final SparkSession spark;
private final Table table;
private final SparkWriteConf writeConf;
@@ -100,12 +100,12 @@ class SparkWriteBuilder implements WriteBuilder, SupportsDynamicOverwrite, Suppo
}
@Override
- public WriteBuilder overwrite(Filter[] filters) {
+ public WriteBuilder overwrite(Predicate[] predicates) {
Preconditions.checkState(
!overwriteFiles, "Cannot overwrite individual files and using filters");
Preconditions.checkState(rewrittenFileSetId == null, "Cannot overwrite and rewrite");
- this.overwriteExpr = SparkFilters.convert(filters);
+ this.overwriteExpr = SparkV2Filters.convert(predicates);
if (overwriteExpr == Expressions.alwaysTrue() && "dynamic".equals(overwriteMode)) {
// use the write option to override truncating the table. use dynamic overwrite instead.
this.overwriteDynamic = true;
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java
index e608a40b72..df4566da0c 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java
@@ -27,27 +27,27 @@ import org.apache.iceberg.UpdateSchema;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.spark.SparkFilters;
import org.apache.iceberg.spark.SparkSchemaUtil;
import org.apache.iceberg.spark.SparkUtil;
+import org.apache.iceberg.spark.SparkV2Filters;
import org.apache.iceberg.spark.SparkWriteConf;
import org.apache.iceberg.spark.SparkWriteRequirements;
import org.apache.iceberg.types.TypeUtil;
import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.connector.expressions.filter.Predicate;
import org.apache.spark.sql.connector.read.Scan;
import org.apache.spark.sql.connector.write.BatchWrite;
import org.apache.spark.sql.connector.write.LogicalWriteInfo;
import org.apache.spark.sql.connector.write.RowLevelOperation.Command;
import org.apache.spark.sql.connector.write.SupportsDynamicOverwrite;
-import org.apache.spark.sql.connector.write.SupportsOverwrite;
+import org.apache.spark.sql.connector.write.SupportsOverwriteV2;
import org.apache.spark.sql.connector.write.Write;
import org.apache.spark.sql.connector.write.WriteBuilder;
import org.apache.spark.sql.connector.write.streaming.StreamingWrite;
-import org.apache.spark.sql.sources.Filter;
import org.apache.spark.sql.types.LongType$;
import org.apache.spark.sql.types.StructType;
-class SparkWriteBuilder implements WriteBuilder, SupportsDynamicOverwrite, SupportsOverwrite {
+class SparkWriteBuilder implements WriteBuilder, SupportsDynamicOverwrite, SupportsOverwriteV2 {
private final SparkSession spark;
private final Table table;
private final SparkWriteConf writeConf;
@@ -100,12 +100,12 @@ class SparkWriteBuilder implements WriteBuilder, SupportsDynamicOverwrite, Suppo
}
@Override
- public WriteBuilder overwrite(Filter[] filters) {
+ public WriteBuilder overwrite(Predicate[] predicates) {
Preconditions.checkState(
!overwriteFiles, "Cannot overwrite individual files and using filters");
Preconditions.checkState(rewrittenFileSetId == null, "Cannot overwrite and rewrite");
- this.overwriteExpr = SparkFilters.convert(filters);
+ this.overwriteExpr = SparkV2Filters.convert(predicates);
if (overwriteExpr == Expressions.alwaysTrue() && "dynamic".equals(overwriteMode)) {
// use the write option to override truncating the table. use dynamic overwrite instead.
this.overwriteDynamic = true;
diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java
index 89af7740d9..c1867433fd 100644
--- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java
+++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java
@@ -27,27 +27,27 @@ import org.apache.iceberg.UpdateSchema;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.spark.SparkFilters;
import org.apache.iceberg.spark.SparkSchemaUtil;
import org.apache.iceberg.spark.SparkUtil;
+import org.apache.iceberg.spark.SparkV2Filters;
import org.apache.iceberg.spark.SparkWriteConf;
import org.apache.iceberg.spark.SparkWriteRequirements;
import org.apache.iceberg.types.TypeUtil;
import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.connector.expressions.filter.Predicate;
import org.apache.spark.sql.connector.read.Scan;
import org.apache.spark.sql.connector.write.BatchWrite;
import org.apache.spark.sql.connector.write.LogicalWriteInfo;
import org.apache.spark.sql.connector.write.RowLevelOperation.Command;
import org.apache.spark.sql.connector.write.SupportsDynamicOverwrite;
-import org.apache.spark.sql.connector.write.SupportsOverwrite;
+import org.apache.spark.sql.connector.write.SupportsOverwriteV2;
import org.apache.spark.sql.connector.write.Write;
import org.apache.spark.sql.connector.write.WriteBuilder;
import org.apache.spark.sql.connector.write.streaming.StreamingWrite;
-import org.apache.spark.sql.sources.Filter;
import org.apache.spark.sql.types.LongType$;
import org.apache.spark.sql.types.StructType;
-class SparkWriteBuilder implements WriteBuilder, SupportsDynamicOverwrite, SupportsOverwrite {
+class SparkWriteBuilder implements WriteBuilder, SupportsDynamicOverwrite, SupportsOverwriteV2 {
private final SparkSession spark;
private final Table table;
private final SparkWriteConf writeConf;
@@ -100,12 +100,12 @@ class SparkWriteBuilder implements WriteBuilder, SupportsDynamicOverwrite, Suppo
}
@Override
- public WriteBuilder overwrite(Filter[] filters) {
+ public WriteBuilder overwrite(Predicate[] predicates) {
Preconditions.checkState(
!overwriteFiles, "Cannot overwrite individual files and using filters");
Preconditions.checkState(rewrittenFileSetId == null, "Cannot overwrite and rewrite");
- this.overwriteExpr = SparkFilters.convert(filters);
+ this.overwriteExpr = SparkV2Filters.convert(predicates);
if (overwriteExpr == Expressions.alwaysTrue() && "dynamic".equals(overwriteMode)) {
// use the write option to override truncating the table. use dynamic overwrite instead.
this.overwriteDynamic = true;