You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by at...@apache.org on 2026/06/18 01:53:19 UTC
(iceberg-go) branch main updated: test(recipe): bump Spark 4 integration Iceberg runtime to 1.11.0 (#1224)
This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 5a94890f test(recipe): bump Spark 4 integration Iceberg runtime to 1.11.0 (#1224)
5a94890f is described below
commit 5a94890fee62c643c8c26e0190f9d064f237d038
Author: Neelesh Salian <ns...@users.noreply.github.com>
AuthorDate: Wed Jun 17 18:53:15 2026 -0700
test(recipe): bump Spark 4 integration Iceberg runtime to 1.11.0 (#1224)
## Changes
- Bumps the Spark 4 integration recipe's Iceberg runtime 1.10.0 ->
1.11.0, which adds shredded variant write support
(write.parquet.shred-variants, apache/iceberg
[#14297](https://github.com/apache/iceberg/pull/14297)) - a prerequisite
for the shredding implementation in `iceberg-go`
[#986](https://github.com/apache/iceberg-go/issues/986) and
[#987](https://github.com/apache/iceberg-go/issues/987)
- provision.py: qualify the Iceberg ALTER extensions (ADD PARTITION
FIELD / CREATE TAG / CREATE BRANCH) as spark_catalog.default.* so
segment 0 resolves as the configured catalog. 1.11.0 added a WARN
(apache/iceberg [#14183](https://github.com/apache/iceberg/pull/14183))
when Spark3Util probes a bare "default" as a catalog
## Test plan
- locally ran both the Spark 3.5 and Spark 4 integration test suites.
---
internal/recipe/Dockerfile.spark4 | 2 +-
internal/recipe/provision.py | 13 ++++++++-----
2 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/internal/recipe/Dockerfile.spark4 b/internal/recipe/Dockerfile.spark4
index 1361a8f0..6a47e8d5 100644
--- a/internal/recipe/Dockerfile.spark4
+++ b/internal/recipe/Dockerfile.spark4
@@ -17,7 +17,7 @@ FROM apache/spark:4.0.2-python3
USER root
-ARG ICEBERG_VERSION=1.10.0
+ARG ICEBERG_VERSION=1.11.0
ARG SCALA_VERSION=2.13
RUN curl --progress-bar -L --retry 3 \
diff --git a/internal/recipe/provision.py b/internal/recipe/provision.py
index 0cec3f2a..8a822716 100644
--- a/internal/recipe/provision.py
+++ b/internal/recipe/provision.py
@@ -24,6 +24,9 @@ from pyiceberg.types import FixedType, NestedField, UUIDType
spark = SparkSession.builder.getOrCreate()
+# Spark 4 (Iceberg 1.11+) warns on a bare "default." in ALTER extensions; Spark 3.5 (Iceberg 1.8) can't plan them with an explicit catalog. Qualify only on 4+.
+alter_ns = "spark_catalog.default" if int(spark.version.split(".")[0]) >= 4 else "default"
+
catalogs = {
'rest': load_catalog(
"rest",
@@ -138,7 +141,7 @@ for catalog_name, catalog in catalogs.items():
# Partitioning is not really needed, but there is a bug:
# https://github.com/apache/iceberg/pull/7685
- spark.sql(f"ALTER TABLE default.test_positional_mor_deletes ADD PARTITION FIELD years(dt) AS dt_years")
+ spark.sql(f"ALTER TABLE {alter_ns}.test_positional_mor_deletes ADD PARTITION FIELD years(dt) AS dt_years")
spark.sql(
f"""
@@ -159,9 +162,9 @@ for catalog_name, catalog in catalogs.items():
"""
)
- spark.sql(f"ALTER TABLE default.test_positional_mor_deletes CREATE TAG tag_12")
+ spark.sql(f"ALTER TABLE {alter_ns}.test_positional_mor_deletes CREATE TAG tag_12")
- spark.sql(f"ALTER TABLE default.test_positional_mor_deletes CREATE BRANCH without_5")
+ spark.sql(f"ALTER TABLE {alter_ns}.test_positional_mor_deletes CREATE BRANCH without_5")
# WAP must be enabled for spark.wap.branch to scope DML to the branch
spark.sql(f"ALTER TABLE default.test_positional_mor_deletes SET TBLPROPERTIES ('write.wap.enabled'='true')")
@@ -189,7 +192,7 @@ for catalog_name, catalog in catalogs.items():
"""
)
- spark.sql(f"ALTER TABLE default.test_positional_mor_double_deletes ADD PARTITION FIELD years(dt) AS dt_years")
+ spark.sql(f"ALTER TABLE {alter_ns}.test_positional_mor_double_deletes ADD PARTITION FIELD years(dt) AS dt_years")
spark.sql(
f"""
@@ -272,7 +275,7 @@ for catalog_name, catalog in catalogs.items():
"""
)
- spark.sql(f"ALTER TABLE default.{table_name} ADD PARTITION FIELD {partition}")
+ spark.sql(f"ALTER TABLE {alter_ns}.{table_name} ADD PARTITION FIELD {partition}")
spark.sql(
f"""