You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by yz...@apache.org on 2026/06/09 01:47:49 UTC
(polaris) 04/12: spark v4.0: adapt for Spark 4.0 API and runtime changes
This is an automated email from the ASF dual-hosted git repository.
MonkeyCanCode pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
commit 7f93538e5fd6e94eca3f808fcef3776fa58d9794
Author: Yong <yo...@gmail.com>
AuthorDate: Thu Jun 4 00:36:28 2026 -0500
spark v4.0: adapt for Spark 4.0 API and runtime changes
---
CHANGELOG.md | 1 +
gradle/libs.versions.toml | 5 +-
plugins/spark/README.md | 21 +-
plugins/spark/spark-scala.properties | 6 +-
plugins/spark/v4.0/getting-started/README.md | 9 +-
.../spark/v4.0/getting-started/docker-compose.yml | 2 +-
.../v4.0/getting-started/notebooks/Dockerfile | 10 +-
.../getting-started/notebooks/SparkPolaris.ipynb | 4 +-
plugins/spark/v4.0/integration/build.gradle.kts | 36 ++--
.../spark/quarkus/it/SparkCatalogBaseIT.java | 58 ++---
plugins/spark/v4.0/regtests/Dockerfile | 46 ----
plugins/spark/v4.0/regtests/README.md | 131 ------------
plugins/spark/v4.0/regtests/docker-compose.yml | 50 -----
plugins/spark/v4.0/regtests/run.sh | 210 ------------------
plugins/spark/v4.0/regtests/setup.sh | 238 ---------------------
.../spark/v4.0/regtests/suites/spark_sql_delta.ref | 57 -----
.../spark/v4.0/regtests/suites/spark_sql_delta.sh | 81 -------
.../spark/v4.0/regtests/suites/spark_sql_hudi.ref | 45 ----
.../spark/v4.0/regtests/suites/spark_sql_hudi.sh | 85 --------
plugins/spark/v4.0/spark/BUNDLE-LICENSE | 4 +-
plugins/spark/v4.0/spark/build.gradle.kts | 18 +-
.../spark/v4.0/spark/checkstyle_suppressions.xml | 6 +-
.../org/apache/polaris/spark/SparkCatalog.java | 29 +--
.../polaris/spark/utils/PolarisCatalogUtils.java | 3 +-
.../org/apache/polaris/spark/SparkCatalogTest.java | 89 ++++----
settings.gradle.kts | 4 +-
26 files changed, 163 insertions(+), 1085 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 17c77b54b1..7e0c6d528c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -51,6 +51,7 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
- Python client: added support for Python 3.14
- Added support for `register table` overwrite semantics in the Iceberg REST catalog flow (`overwrite=true`) for internal Polaris catalogs. With overwrite enabled, existing table pointers can be updated to a new metadata location while preserving default behavior for `overwrite=false`.
- Added `REGISTER_TABLE_OVERWRITE` authorization operation mapped to `TABLE_FULL_METADATA` for deterministic overwrite authorization.
+- Added Polaris Spark 4.0 client.
### Changes
- Added REPL support to Polaris CLI.
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 610b3b8825..1abd0b5f16 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -30,6 +30,7 @@ ranger-graalvm = "25.0.3"
scala212 = "2.12.21"
scala213 = "2.13.18"
spark35 = "3.5.6"
+spark40 = "4.0.2"
swagger = "1.6.16"
[bundles]
@@ -41,7 +42,8 @@ swagger = "1.6.16"
# (aka mention of the dependency removed).
#
agrona = { module = "org.agrona:agrona", version = "2.4.1" }
-antlr4-runtime = { module = "org.antlr:antlr4-runtime", version.strictly = "4.9.3" } # spark integration tests
+antlr4-runtime-spark35 = { module = "org.antlr:antlr4-runtime", version.strictly = "4.9.3" } # spark 3.5 integration tests
+antlr4-runtime-spark40 = { module = "org.antlr:antlr4-runtime", version.strictly = "4.13.1" } # spark 4.0 integration tests
apache-httpclient5 = { module = "org.apache.httpcomponents.client5:httpclient5", version = "5.6.1" }
assertj-core = { module = "org.assertj:assertj-core", version = "3.27.7" }
auth0-jwt = { module = "com.auth0:java-jwt", version = "4.5.2" }
@@ -125,4 +127,3 @@ jmh = { id = "me.champeau.jmh", version = "0.7.3" }
openapi-generator = { id = "org.openapi.generator", version = "7.12.0" }
quarkus = { id = "io.quarkus", version.ref = "quarkus" }
rat = { id = "org.nosphere.apache.rat", version = "0.8.1" }
-
diff --git a/plugins/spark/README.md b/plugins/spark/README.md
index 84425a9914..ad791cadb4 100644
--- a/plugins/spark/README.md
+++ b/plugins/spark/README.md
@@ -19,18 +19,22 @@
# Polaris Spark Plugin
+This directory contains the Polaris Spark plugins. The plugins are built for specific versions of Spark:
+- [Spark 3.5](./v3.5/getting-started/README.md)
+- [Spark 4.0](./v4.0/getting-started/README.md)
+
The Polaris Spark plugin provides a SparkCatalog class, which communicates with the Polaris
REST endpoints, and provides implementations for Apache Spark's
- [TableCatalog](https://github.com/apache/spark/blob/v3.5.6/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java)
- [ViewCatalog](https://github.com/apache/spark/blob/v3.5.6/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/ViewCatalog.java)
- [SupportsNamespaces](https://github.com/apache/spark/blob/v3.5.6/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java)
-Right now, the plugin only provides support for Spark 3.5, Scala version 2.12 and 2.13, and depends on iceberg-spark-runtime 1.11.0.
+The plugin provides support for Spark 3.5 (Scala version 2.12 and 2.13) and Spark 4.0 (Scala version 2.13).
The Polaris Spark client supports catalog management for both Iceberg and Delta tables. It routes all Iceberg table
requests to the Iceberg REST endpoints and routes all Delta table requests to the Generic Table REST endpoints.
-The Spark Client requires at least delta 3.2.1 to work with Delta tables, which requires at least Apache Spark 3.5.3.
+The Spark Client requires at least delta 3.2.1 to work with Delta tables for Spark 3.5, and delta 4.0.1 for Spark 4.0.
# Start Spark with local Polaris service using the Polaris Spark plugin
The following command starts a Polaris server for local testing, it runs on localhost:8181 with default
@@ -45,8 +49,8 @@ option with the Polaris Spark package, or the `--jars` option with the Polaris S
The following sections explain how to build and run Spark with both the Polaris package and the bundle JAR.
# Build and run with Polaris spark package locally
-The Polaris Spark client source code is located in plugins/spark/v3.5/spark. To use the Polaris Spark package
-with Spark, you first need to publish the source JAR to your local Maven repository.
+The Polaris Spark client source code is located in plugins/spark/v<spark_version>/spark. To use the Polaris Spark
+package with Spark, you first need to publish the source JAR to your local Maven repository.
Run the following command to build the Polaris Spark project and publish the source JAR to your local Maven repository:
- `./gradlew assemble` -- build the whole Polaris project without running tests
@@ -54,7 +58,7 @@ Run the following command to build the Polaris Spark project and publish the sou
```shell
bin/spark-shell \
---packages org.apache.polaris:polaris-spark-<spark_version>_<scala_version>:<polaris_version>,org.apache.iceberg:iceberg-aws-bundle:1.11.0,io.delta:delta-spark_2.12:3.3.1 \
+--packages org.apache.polaris:polaris-spark-<spark_version>_<scala_version>:<polaris_version>,org.apache.iceberg:iceberg-aws-bundle:1.11.0,io.delta:delta-spark_<scala_version>:<delta_version> \
--conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions,io.delta.sql.DeltaSparkSessionExtension \
--conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog \
--conf spark.sql.catalog.<catalog-name>.warehouse=<catalog-name> \
@@ -71,13 +75,14 @@ The Polaris version is defined in the `versions.txt` file located in the root di
Assume the following values:
- `spark_version`: 3.5
- `scala_version`: 2.12
-- `polaris_version`: 1.2.0-incubating-SNAPSHOT
+- `delta_version`: 3.3.1
+- `polaris_version`: 1.6.0-SNAPSHOT
- `catalog-name`: `polaris`
The Spark command would look like following:
```shell
bin/spark-shell \
---packages org.apache.polaris:polaris-spark-3.5_2.12:1.2.0-incubating-SNAPSHOT,org.apache.iceberg:iceberg-aws-bundle:1.11.0,io.delta:delta-spark_2.12:3.3.1 \
+--packages org.apache.polaris:polaris-spark-3.5_2.12:1.6.0-SNAPSHOT,org.apache.iceberg:iceberg-aws-bundle:1.11.0,io.delta:delta-spark_2.12:3.3.1 \
--conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions,io.delta.sql.DeltaSparkSessionExtension \
--conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog \
--conf spark.sql.catalog.polaris.warehouse=polaris \
@@ -94,7 +99,7 @@ bin/spark-shell \
The polaris-spark project also provides a Spark bundle JAR for the `--jars` use case. The resulting JAR will follow this naming format:
polaris-spark-<spark_version>_<scala_version>-<polaris_version>-bundle.jar
For example:
-polaris-spark-bundle-3.5_2.12-1.2.0-incubating-SNAPSHOT-bundle.jar
+polaris-spark-3.5_2.12-1.6.0-SNAPSHOT-bundle.jar
Run `./gradlew assemble` to build the entire Polaris project without running tests. After the build completes,
the bundle JAR can be found under: plugins/spark/v3.5/spark/build/<scala_version>/libs/.
diff --git a/plugins/spark/spark-scala.properties b/plugins/spark/spark-scala.properties
index 2ed71b574f..24de347d78 100644
--- a/plugins/spark/spark-scala.properties
+++ b/plugins/spark/spark-scala.properties
@@ -17,6 +17,8 @@
# under the License.
#
-sparkVersions=3.5
+sparkVersions=3.5,4.0
-scalaVersions=2.12,2.13
+scalaVersions.3.5=2.12,2.13
+
+scalaVersions.4.0=2.13
diff --git a/plugins/spark/v4.0/getting-started/README.md b/plugins/spark/v4.0/getting-started/README.md
index 582bd177a8..148a3f4524 100644
--- a/plugins/spark/v4.0/getting-started/README.md
+++ b/plugins/spark/v4.0/getting-started/README.md
@@ -28,7 +28,7 @@ A Jupyter notebook is started to run PySpark, and Polaris Python client is also
directly through Python Client.
## Build the Spark Client Jar and Polaris image
-If Spark Client Jar is not presented locally under plugins/spark/v3.5/build/<scala_version>/libs, please build the jar
+If Spark Client Jar is not presented locally under plugins/spark/v4.0/spark/build/2.13/libs, please build the jar
using
- `./gradlew assemble` -- build the Polaris project and skip the tests.
@@ -45,16 +45,15 @@ If a Polaris image is not already present locally, build one with the following
To start the `docker-compose` file, run this command from the repo's root directory:
```shell
-docker-compose -f plugins/spark/v3.5/getting-started/docker-compose.yml up
+docker-compose -f plugins/spark/v4.0/getting-started/docker-compose.yml up
```
This will spin up 2 container services
* The `polaris` service for running Apache Polaris using an in-memory metastore
* The `jupyter` service for running Jupyter notebook with PySpark
-NOTE: Starting the container first time may take a couple of minutes, because it will need to download the Spark 3.5.6.
-When working with Delta, the Polaris Spark Client requires delta-io >= 3.2.1, and it requires at least Spark 3.5.3,
-but the current jupyter Spark image only support Spark 3.5.0.
+NOTE: Starting the container first time may take a couple of minutes, because it will need to download the Spark 4.0.2.
+When working with Delta, the Polaris Spark client requires delta-io >= 4.0.1, and it requires at least Spark 4.0.2.
### Run with AWS access setup
If you want to interact with S3 bucket, make sure you have the following environment variables setup correctly in
diff --git a/plugins/spark/v4.0/getting-started/docker-compose.yml b/plugins/spark/v4.0/getting-started/docker-compose.yml
index fd7003936c..3e29dd87fa 100644
--- a/plugins/spark/v4.0/getting-started/docker-compose.yml
+++ b/plugins/spark/v4.0/getting-started/docker-compose.yml
@@ -41,7 +41,7 @@ services:
jupyter:
build:
context: ../../../../ # this is needed to get the ./client
- dockerfile: ./plugins/spark/v3.5/getting-started/notebooks/Dockerfile
+ dockerfile: ./plugins/spark/v4.0/getting-started/notebooks/Dockerfile
network: host
ports:
- "8888:8888"
diff --git a/plugins/spark/v4.0/getting-started/notebooks/Dockerfile b/plugins/spark/v4.0/getting-started/notebooks/Dockerfile
index aed50285a6..cc855d6b84 100644
--- a/plugins/spark/v4.0/getting-started/notebooks/Dockerfile
+++ b/plugins/spark/v4.0/getting-started/notebooks/Dockerfile
@@ -17,17 +17,19 @@
# under the License.
#
-FROM docker.io/apache/spark:3.5.6-java17
+FROM docker.io/apache/spark:4.0.2-scala2.13-java17-ubuntu
ENV PYSPARK_PYTHON=/home/spark/venv/bin/python \
- PYTHONPATH="${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-0.10.9.7-src.zip:/home/spark/venv/lib/python3.10/site-packages"
+ PYTHONPATH="${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-0.10.9.9-src.zip:/home/spark/venv/lib/python3.10/site-packages"
USER root
+# Spark 4.0 ubuntu image creates the spark user with HOME=/nonexistent, thus, we need to run usermod to assign the HOME
RUN apt-get update -y && \
apt-get install -y python3-venv && \
mkdir -p /home/spark && \
- chown -R spark /home/spark
+ chown -R spark /home/spark && \
+ usermod -d /home/spark spark
USER spark
@@ -36,7 +38,7 @@ WORKDIR /home/spark
COPY --chown=spark client /home/spark/client
COPY --chown=spark regtests/requirements.txt /tmp
COPY --chown=spark regtests/notebook_requirements.txt /tmp
-COPY --chown=spark plugins/spark/v3.5/spark/build/2.12/libs/*bundle.jar /opt/spark/jars/
+COPY --chown=spark plugins/spark/v4.0/spark/build/2.13/libs/*bundle.jar /opt/spark/jars/
RUN python3 -m venv /home/spark/venv && \
. /home/spark/venv/bin/activate && \
diff --git a/plugins/spark/v4.0/getting-started/notebooks/SparkPolaris.ipynb b/plugins/spark/v4.0/getting-started/notebooks/SparkPolaris.ipynb
index cc2f62ddb2..086732eb12 100644
--- a/plugins/spark/v4.0/getting-started/notebooks/SparkPolaris.ipynb
+++ b/plugins/spark/v4.0/getting-started/notebooks/SparkPolaris.ipynb
@@ -265,9 +265,7 @@
"from pyspark.sql import SparkSession\n",
"\n",
"spark = (SparkSession.builder\n",
- " # This jar is now automatically discovered, thus no longer needed\n",
- " #.config(\"spark.jars\", \"../polaris_libs/polaris-spark-3.5_2.12-1.2.0-incubating-SNAPSHOT-bundle.jar\")\n",
- " .config(\"spark.jars.packages\", \"org.apache.iceberg:iceberg-aws-bundle:1.11.0,io.delta:delta-spark_2.12:3.2.1\")\n",
+ " .config(\"spark.jars.packages\", \"org.apache.iceberg:iceberg-aws-bundle:1.11.0,io.delta:delta-spark_2.13:4.0.1\")\n",
" .config(\"spark.sql.catalog.spark_catalog\", \"org.apache.spark.sql.delta.catalog.DeltaCatalog\")\n",
" .config('spark.sql.iceberg.vectorization.enabled', 'false')\n",
"\n",
diff --git a/plugins/spark/v4.0/integration/build.gradle.kts b/plugins/spark/v4.0/integration/build.gradle.kts
index 6929bcae7b..acd933132d 100644
--- a/plugins/spark/v4.0/integration/build.gradle.kts
+++ b/plugins/spark/v4.0/integration/build.gradle.kts
@@ -24,16 +24,11 @@ plugins {
}
// get version information
-val sparkMajorVersion = "3.5"
+val sparkMajorVersion = "4.0"
val scalaVersion = getAndUseScalaVersionForProject()
val icebergVersion = libs.versions.iceberg.get()
-val spark35Version = libs.versions.spark35.get()
-val scalaLibraryVersion =
- if (scalaVersion == "2.12") {
- libs.versions.scala212.get()
- } else {
- libs.versions.scala213.get()
- }
+val spark40Version = libs.versions.spark40.get()
+val scalaLibraryVersion = libs.versions.scala213.get()
sourceSets {
named("intTest") {
@@ -50,6 +45,12 @@ dependencies {
exclude(group = "org.scala-lang", module = "scala-reflect")
}
+ // For test configurations, exclude jakarta.servlet-api from Quarkus BOM
+ // to allow Spark 4.0's version (5.0.0) which includes SingleThreadModel
+ testImplementation(platform(libs.quarkus.bom)) {
+ exclude(group = "jakarta.servlet", module = "jakarta.servlet-api")
+ }
+
implementation(project(":polaris-runtime-service"))
testImplementation(
@@ -61,7 +62,7 @@ dependencies {
testImplementation(project(":polaris-runtime-test-common"))
- testImplementation("org.apache.spark:spark-sql_${scalaVersion}:${spark35Version}") {
+ testImplementation("org.apache.spark:spark-sql_${scalaVersion}:${spark40Version}") {
// exclude log4j dependencies. Explicit dependencies for the log4j libraries are
// enforced below to ensure the version compatibility
exclude("org.apache.logging.log4j", "log4j-slf4j2-impl")
@@ -71,7 +72,7 @@ dependencies {
}
// Add spark-hive for Hudi integration - provides HiveExternalCatalog that Hudi needs
- testRuntimeOnly("org.apache.spark:spark-hive_${scalaVersion}:${spark35Version}") {
+ testRuntimeOnly("org.apache.spark:spark-hive_${scalaVersion}:${spark40Version}") {
// exclude log4j dependencies to match spark-sql exclusions
exclude("org.apache.logging.log4j", "log4j-slf4j2-impl")
exclude("org.apache.logging.log4j", "log4j-1.2-api")
@@ -84,8 +85,8 @@ dependencies {
// of spark-sql dependency
testRuntimeOnly("org.apache.logging.log4j:log4j-core:2.26.0")
- testImplementation("io.delta:delta-spark_${scalaVersion}:3.3.1")
- testImplementation("org.apache.hudi:hudi-spark3.5-bundle_${scalaVersion}:1.1.1") {
+ testImplementation("io.delta:delta-spark_${scalaVersion}:4.0.1")
+ testImplementation("org.apache.hudi:hudi-spark4.0-bundle_${scalaVersion}:1.1.1") {
// exclude log4j dependencies to match spark-sql exclusions
// exclude log4j dependencies to match spark-sql exclusions and prevent version conflicts
exclude("org.apache.logging.log4j", "log4j-slf4j2-impl")
@@ -102,7 +103,7 @@ dependencies {
// The hudi-spark-bundle includes most Hive libraries but excludes hive-exec to keep size
// manageable
- // This matches what Spark 3.5 distribution provides (hive-exec-2.3.10-core.jar)
+ // This matches what Spark 4.0 distribution provides (hive-exec-2.3.10-core.jar)
testImplementation("org.apache.hive:hive-exec:2.3.10:core") {
// Exclude conflicting dependencies to use Spark's versions
exclude("org.apache.hadoop", "*")
@@ -138,7 +139,14 @@ dependencies {
testImplementation(enforcedPlatform("org.scala-lang:scala-library:${scalaLibraryVersion}"))
testImplementation(enforcedPlatform("org.scala-lang:scala-reflect:${scalaLibraryVersion}"))
testImplementation(libs.javax.servlet.api)
- testImplementation(libs.antlr4.runtime.spark35)
+ testImplementation(libs.antlr4.runtime.spark40)
+}
+
+// Force jakarta.servlet-api to 5.0.0 for Spark 4.0 compatibility
+// Spark 4.0 requires version 5.0.0 which includes SingleThreadModel
+// Quarkus BOM forces it to 6.x which removed SingleThreadModel
+configurations.named("intTestRuntimeClasspath") {
+ resolutionStrategy { force("jakarta.servlet:jakarta.servlet-api:5.0.0") }
}
tasks.named<Test>("intTest").configure {
diff --git a/plugins/spark/v4.0/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java b/plugins/spark/v4.0/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java
index 23f2f1f5fd..76b99d1c14 100644
--- a/plugins/spark/v4.0/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java
+++ b/plugins/spark/v4.0/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java
@@ -39,6 +39,7 @@ import org.apache.spark.sql.connector.catalog.SupportsNamespaces;
import org.apache.spark.sql.connector.catalog.View;
import org.apache.spark.sql.connector.catalog.ViewCatalog;
import org.apache.spark.sql.connector.catalog.ViewChange;
+import org.apache.spark.sql.connector.catalog.ViewInfo;
import org.apache.spark.sql.types.StructType;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -154,15 +155,16 @@ public abstract class SparkCatalogBaseIT extends SparkIntegrationBase {
Identifier viewIdentifier = Identifier.of(namespace, "test-view");
String viewSql = "select id from test-table where id < 3";
viewCatalog.createView(
- viewIdentifier,
- viewSql,
- catalogName,
- namespace,
- schema,
- new String[0],
- new String[0],
- new String[0],
- Maps.newHashMap());
+ new ViewInfo(
+ viewIdentifier,
+ viewSql,
+ catalogName,
+ namespace,
+ schema,
+ new String[0],
+ new String[0],
+ new String[0],
+ Maps.newHashMap()));
// load the view
View view = viewCatalog.loadView(viewIdentifier);
@@ -219,15 +221,16 @@ public abstract class SparkCatalogBaseIT extends SparkIntegrationBase {
String view1Name = "test-view1";
String view1SQL = "select id from test-table where id >= 3";
viewCatalog.createView(
- Identifier.of(l1ns, view1Name),
- view1SQL,
- catalogName,
- l1ns,
- schema,
- new String[0],
- new String[0],
- new String[0],
- Maps.newHashMap());
+ new ViewInfo(
+ Identifier.of(l1ns, view1Name),
+ view1SQL,
+ catalogName,
+ l1ns,
+ schema,
+ new String[0],
+ new String[0],
+ new String[0],
+ Maps.newHashMap()));
// create two views under the l2 namespace
String[] nsl2ViewNames = new String[] {"test-view2", "test-view3"};
String[] nsl2ViewSQLs =
@@ -236,15 +239,16 @@ public abstract class SparkCatalogBaseIT extends SparkIntegrationBase {
};
for (int i = 0; i < nsl2ViewNames.length; i++) {
viewCatalog.createView(
- Identifier.of(l2ns, nsl2ViewNames[i]),
- nsl2ViewSQLs[i],
- catalogName,
- l2ns,
- schema,
- new String[0],
- new String[0],
- new String[0],
- Maps.newHashMap());
+ new ViewInfo(
+ Identifier.of(l2ns, nsl2ViewNames[i]),
+ nsl2ViewSQLs[i],
+ catalogName,
+ l2ns,
+ schema,
+ new String[0],
+ new String[0],
+ new String[0],
+ Maps.newHashMap()));
}
// list views under l1ns
Identifier[] l1Views = viewCatalog.listViews(l1ns);
diff --git a/plugins/spark/v4.0/regtests/Dockerfile b/plugins/spark/v4.0/regtests/Dockerfile
deleted file mode 100755
index 5c4c480f20..0000000000
--- a/plugins/spark/v4.0/regtests/Dockerfile
+++ /dev/null
@@ -1,46 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-FROM docker.io/apache/spark:3.5.6-java17
-
-ARG POLARIS_HOST=polaris \
- CURRENT_SCALA_VERSION=2.12
-
-ENV POLARIS_HOST=${POLARIS_HOST} \
- CURRENT_SCALA_VERSION=${CURRENT_SCALA_VERSION}
-
-USER root
-
-RUN apt-get update && \
- apt-get install -y --no-install-recommends diffutils wget curl && \
- rm -rf /var/lib/apt/lists/* && \
- mkdir -p /home/spark /tmp/polaris-regtests /opt/spark/conf && \
- chown -R spark:spark /home/spark /tmp/polaris-regtests && \
- chmod -R 777 /opt/spark/conf
-
-WORKDIR /home/spark/polaris
-
-COPY --chown=spark:spark ./v3.5 /home/spark/polaris/v3.5
-
-# /home/spark/.../regtests might not be writable in all situations, see https://github.com/apache/polaris/pull/205
-RUN chmod -R 777 /home/spark/polaris/v3.5/regtests
-
-USER spark
-
-ENTRYPOINT ["./v3.5/regtests/run.sh"]
diff --git a/plugins/spark/v4.0/regtests/README.md b/plugins/spark/v4.0/regtests/README.md
deleted file mode 100755
index de3355204c..0000000000
--- a/plugins/spark/v4.0/regtests/README.md
+++ /dev/null
@@ -1,131 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-# End-to-end regression tests
-
-regtests provides basic end-to-end tests for spark_sql using spark client jars.
-
-Regression tests are either run in Docker, using docker-compose to orchestrate the tests, or
-locally.
-
-**NOTE** regtests are supposed to be a light-weight testing to ensure jars can be used to start
-spark and run basic SQL commands. Please use integration for detailed testing.
-
-## Prerequisites
-
-It is recommended to clean the `regtests/output` directory before running tests. This can be done by
-running:
-
-```shell
-rm -rf ./plugins/spark/v3.5/regtests/output && mkdir -p ./plugins/spark/v3.5/regtests/output && chmod -R 777 ./plugins/spark/v3.5/regtests/output
-```
-
-## Run Tests With Docker Compose
-
-Tests can be run with docker-compose using the provided `./plugins/spark/v3.5/regtests/docker-compose.yml` file, as
-follows:
-
-```shell
-./gradlew assemble publishToMavenLocal
-./gradlew \
- :polaris-server:assemble \
- :polaris-server:quarkusAppPartsBuild --rerun \
- -Dquarkus.container-image.build=true
-docker compose -f ./plugins/spark/v3.5/regtests/docker-compose.yml up --build --exit-code-from regtest
-```
-
-In this setup, a Polaris container will be started in a docker-compose group, using the image
-previously built by the Gradle build. Then another container, including a Spark SQL shell, will run
-the tests. The exit code will be the same as the exit code of the Spark container.
-**NOTE** Docker compose only support testing with scala 2.12, because no scala 2.13 image is available
-for spark 3.5. Scala 2.13 will be supported for Spark 4.0.
-
-This is the flow used in CI and should be done locally before pushing to GitHub to ensure that no
-environmental factors contribute to the outcome of the tests.
-
-**Important**: if you are also using minikube, for example to test the Helm chart, you may need to
-_unset_ the Docker environment that was pointing to the Minikube Docker daemon, otherwise the image
-will be built by the Minikube Docker daemon and will not be available to the local Docker daemon.
-This can be done by running, _before_ building the image and running the tests:
-
-```shell
-eval $(minikube -p minikube docker-env --unset)
-```
-
-## Run Tests Locally
-
-Regression tests can be run locally as well, using the test harness. For local testing, both
-Scala 2.12 and Scala 2.13 are supported.
-
-To run regression tests locally, run the following:
-- `./gradlew assemble publishToMavenLocal` -- build the Polaris project and Spark Client jars. Publish the binary to local maven repo.
-- `./gradlew run` -- start a Polaris server on localhost:8181.
-- `env POLARIS_HOST=localhost ./plugins/spark/v3.5/regtests/run.sh` -- run regtests.
-
-Note: the regression tests expect Polaris to run with certain options, e.g. with support for `FILE`
-storage, default realm `POLARIS` and root credentials `root:secret`; if you run the above command,
-this will be the case. If you run Polaris in a different way, make sure that Polaris is configured
-appropriately.
-
-## Running Specific Test Suites
-
-By default, `run.sh` auto-discovers and executes all test suites in the `suites/` directory.
-To run a specific suite, use the `REGTEST_SUITE` environment variable with just the test name:
-
-```bash
-# Run only Delta tests
-env POLARIS_HOST=localhost REGTEST_SUITE=spark_sql_delta ./plugins/spark/v3.5/regtests/run.sh
-
-# Run only Hudi tests
-env POLARIS_HOST=localhost REGTEST_SUITE=spark_sql_hudi ./plugins/spark/v3.5/regtests/run.sh
-```
-
-## Adding a New Test Suite
-
-Test suites are auto-discovered from the `suites/` directory. To add a new test:
-
-1. Create `suites/<descriptive_name>_<table_format>.sh` (must be executable)
-2. Create `suites/<descriptive_name>_<table_format>.ref` (expected output)
-3. The table format is automatically parsed from the last segment before `.sh`
-4. Supported table formats: `delta`, `hudi`
-
-## Table Format Support
-
-The regression tests support multiple table formats through the `--tableFormat` parameter in `setup.sh`:
-
-- **Delta** (default): Uses `DeltaCatalog` for `spark_catalog`. Tests both Iceberg and Delta tables.
-- **Hudi**: Uses `HoodieCatalog` for `spark_catalog`. Tests both Iceberg and Hudi tables.
-
-Each test suite is isolated with its own Spark configuration and catalog setup. The `spark_catalog`
-can only be configured to one catalog implementation at a time, which is why separate test suites
-are needed for Delta and Hudi formats.
-
-### Manual Setup
-
-You can manually run `setup.sh` with a specific table format:
-
-```bash
-# Setup for Delta tables (default)
-./plugins/spark/v3.5/regtests/setup.sh --sparkVersion 3.5.6 --scalaVersion 2.12 --polarisVersion 0.1.0 --tableFormat delta
-
-# Setup for Hudi tables
-./plugins/spark/v3.5/regtests/setup.sh --sparkVersion 3.5.6 --scalaVersion 2.12 --polarisVersion 0.1.0 --tableFormat hudi
-```
diff --git a/plugins/spark/v4.0/regtests/docker-compose.yml b/plugins/spark/v4.0/regtests/docker-compose.yml
deleted file mode 100755
index 5c9c2ea363..0000000000
--- a/plugins/spark/v4.0/regtests/docker-compose.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-services:
- polaris:
- image: apache/polaris:latest
- ports:
- - "8181"
- - "8182"
- environment:
- AWS_REGION: us-west-2
- POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,root,s3cr3t
- quarkus.log.file.enabled: "false"
- quarkus.otel.sdk.disabled: "true"
- polaris.features."ALLOW_INSECURE_STORAGE_TYPES": "true"
- polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES": "[\"FILE\",\"S3\",\"GCS\",\"AZURE\"]"
- polaris.readiness.ignore-severe-issues: "true"
- healthcheck:
- test: ["CMD", "curl", "http://localhost:8182/q/health"]
- interval: 10s
- timeout: 10s
- retries: 5
- regtest:
- build:
- context: ../..
- dockerfile: v3.5/regtests/Dockerfile
- args:
- POLARIS_HOST: polaris
- depends_on:
- polaris:
- condition: service_healthy
- volumes:
- - ~/.m2:/home/spark/.m2
- - ./output:/tmp/polaris-regtests/
diff --git a/plugins/spark/v4.0/regtests/run.sh b/plugins/spark/v4.0/regtests/run.sh
deleted file mode 100755
index d925f7e75c..0000000000
--- a/plugins/spark/v4.0/regtests/run.sh
+++ /dev/null
@@ -1,210 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# Run without args to run all tests.
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-SPARK_ROOT_DIR=$(dirname ${SCRIPT_DIR})
-export SPARK_LOCAL_HOSTNAME=localhost # avoid VPN messing up driver local IP address binding
-
-FMT_RED='\033[0;31m'
-FMT_GREEN='\033[0;32m'
-FMT_NC='\033[0m'
-
-function loginfo() {
- echo "$(date): ${@}"
-}
-function loggreen() {
- echo -e "${FMT_GREEN}$(date): ${@}${FMT_NC}"
-}
-function logred() {
- echo -e "${FMT_RED}$(date): ${@}${FMT_NC}"
-}
-
-# Allow bearer token to be provided if desired
-if [[ -z "$REGTEST_ROOT_BEARER_TOKEN" ]]; then
- if ! output=$(curl -X POST -H "Polaris-Realm: POLARIS" "http://${POLARIS_HOST:-localhost}:8181/api/catalog/v1/oauth/tokens" \
- -d "grant_type=client_credentials" \
- -d "client_id=root" \
- -d "client_secret=s3cr3t" \
- -d "scope=PRINCIPAL_ROLE:ALL"); then
- logred "Error: Failed to retrieve bearer token"
- exit 1
- fi
-
- token=$(echo "$output" | awk -F\" '{print $4}')
-
- if [ "$token" == "unauthorized_client" ]; then
- logred "Error: Failed to retrieve bearer token"
- exit 1
- fi
-
- export REGTEST_ROOT_BEARER_TOKEN=$token
-fi
-
-echo "Root bearer token: ${REGTEST_ROOT_BEARER_TOKEN}"
-
-NUM_FAILURES=0
-
-SCALA_VERSIONS=("2.12" "2.13")
-if [[ -n "$CURRENT_SCALA_VERSION" ]]; then
- SCALA_VERSIONS=("${CURRENT_SCALA_VERSION}")
-fi
-SPARK_MAJOR_VERSION="3.5"
-SPARK_VERSION="3.5.6"
-
-SPARK_SHELL_OPTIONS=("PACKAGE" "JAR")
-
-# Auto-discover test suites from the suites/ directory
-# Test files must follow naming convention: <name>_<table_format>.sh
-SUITES_DIR="${SCRIPT_DIR}/suites"
-
-if [[ ! -d "$SUITES_DIR" ]]; then
- logred "Error: Test suites directory not found: ${SUITES_DIR}"
- exit 1
-fi
-
-# Parses a test suite filename (e.g. "spark_sql_delta.sh") to extract:
-# TABLE_FORMAT - the table format suffix after the last '_' (e.g. "delta")
-# TEST_SHORTNAME - the base name without the .sh extension (e.g. "spark_sql_delta")
-# TEST_FILE - the full path to the suite file under SUITES_DIR
-parse_test_suite() {
- local filename="$1"
- local base="${filename%.sh}"
- TABLE_FORMAT="${base##*_}"
- TEST_SHORTNAME="${base}"
- TEST_FILE="${SUITES_DIR}/${filename}"
-}
-
-declare -a TEST_SUITES=()
-for test_file in "${SUITES_DIR}"/*.sh; do
- [[ -f "$test_file" ]] || continue
- TEST_SUITES+=("$(basename "$test_file")")
-done
-
-if [[ ${#TEST_SUITES[@]} -eq 0 ]]; then
- logred "Error: No test suites found in ${SUITES_DIR}"
- exit 1
-fi
-
-# Allow running specific test via environment variable
-echo "REGTEST_SUITE=${REGTEST_SUITE}"
-if [[ -n "$REGTEST_SUITE" ]]; then
- REGTEST_SUITE="${REGTEST_SUITE%.sh}"
- SUITE_FILE="${REGTEST_SUITE}.sh"
- if [[ ! -f "${SUITES_DIR}/${SUITE_FILE}" ]]; then
- logred "Error: Test suite not found: ${SUITES_DIR}/${SUITE_FILE}"
- exit 1
- fi
- echo "Overriding TEST_SUITES to run only: ${REGTEST_SUITE}"
- TEST_SUITES=("${SUITE_FILE}")
-fi
-echo "Will run test suites: ${TEST_SUITES[@]}"
-
-for SCALA_VERSION in "${SCALA_VERSIONS[@]}"; do
- echo "RUN REGRESSION TEST FOR SPARK_MAJOR_VERSION=${SPARK_MAJOR_VERSION}, SPARK_VERSION=${SPARK_VERSION}, SCALA_VERSION=${SCALA_VERSION}"
- # find the project jar
- SPARK_DIR=${SPARK_ROOT_DIR}/spark
- JAR_PATH=$(find ${SPARK_DIR} -name "polaris-spark-${SPARK_MAJOR_VERSION}_${SCALA_VERSION}-*.*-bundle.jar" -print -quit)
- echo "find jar ${JAR_PATH}"
-
- # extract the polaris
- JAR_NAME=$(basename "$JAR_PATH")
- echo "JAR_NAME=${JAR_NAME}"
- POLARIS_VERSION=$(echo "$JAR_NAME" | sed -n 's/.*-\([0-9][^-]*.*\)-bundle\.jar/\1/p')
- echo "$POLARIS_VERSION"
-
- SPARK_EXISTS="TRUE"
- if [ -z "${SPARK_HOME}" ]; then
- SPARK_EXISTS="FALSE"
- fi
-
- for SPARK_SHELL_OPTION in "${SPARK_SHELL_OPTIONS[@]}"; do
- # Loop through each test suite
- for TEST_SUITE_FILE in "${TEST_SUITES[@]}"; do
- parse_test_suite "$TEST_SUITE_FILE"
-
- loginfo "Setting up for test suite: ${TEST_SHORTNAME} with table format: ${TABLE_FORMAT}"
-
- # clean up the default configuration if exists
- if [ -d "${SPARK_HOME}" ]; then
- SPARK_CONF="${SPARK_HOME}/conf/spark-defaults.conf"
- if [ -f "${SPARK_CONF}" ]; then
- echo "Clean spark conf file"
- rm ${SPARK_CONF}
- fi
- fi
-
- echo "finish SPARK_HOME check"
-
- # Run setup with appropriate table format
- if [ "${SPARK_SHELL_OPTION}" == "PACKAGE" ]; then
- # run the setup without jar configuration
- source ${SCRIPT_DIR}/setup.sh --sparkVersion ${SPARK_VERSION} --scalaVersion ${SCALA_VERSION} --polarisVersion ${POLARIS_VERSION} --tableFormat ${TABLE_FORMAT}
- else
- source ${SCRIPT_DIR}/setup.sh --sparkVersion ${SPARK_VERSION} --scalaVersion ${SCALA_VERSION} --polarisVersion ${POLARIS_VERSION} --jar ${JAR_PATH} --tableFormat ${TABLE_FORMAT}
- fi
-
- # run the test
- loginfo "Starting test ${TEST_SHORTNAME}"
-
- TEST_TMPDIR="/tmp/polaris-spark-regtests/${TEST_SHORTNAME}_${SPARK_MAJOR_VERSION}_${SCALA_VERSION}_${SPARK_SHELL_OPTION}_${TABLE_FORMAT}"
- TEST_STDERR="${TEST_TMPDIR}/${TEST_SHORTNAME}.stderr"
- TEST_STDOUT="${TEST_TMPDIR}/${TEST_SHORTNAME}.stdout"
-
- mkdir -p ${TEST_TMPDIR}
- if (( ${VERBOSE} )); then
- ${TEST_FILE} 2>${TEST_STDERR} | grep -v 'loading settings' | tee ${TEST_STDOUT}
- else
- ${TEST_FILE} 2>${TEST_STDERR} | grep -v 'loading settings' > ${TEST_STDOUT}
- fi
- loginfo "Test run concluded for ${TEST_SHORTNAME}"
-
- # Compare output with reference
- TEST_REF="${SUITES_DIR}/${TEST_SHORTNAME}.ref"
- if cmp --silent ${TEST_STDOUT} ${TEST_REF}; then
- loggreen "Test SUCCEEDED: ${TEST_SHORTNAME}"
- else
- logred "Test FAILED: ${TEST_SHORTNAME}"
- echo '#!/bin/bash' > ${TEST_TMPDIR}/${TEST_SHORTNAME}.fixdiffs.sh
- echo "meld ${TEST_STDOUT} ${TEST_REF}" >> ${TEST_TMPDIR}/${TEST_SHORTNAME}.fixdiffs.sh
- chmod 750 ${TEST_TMPDIR}/${TEST_SHORTNAME}.fixdiffs.sh
- logred "To compare and fix diffs (if 'meld' installed): ${TEST_TMPDIR}/${TEST_SHORTNAME}.fixdiffs.sh"
- logred "Or manually diff: diff ${TEST_STDOUT} ${TEST_REF}"
- logred "See stderr from test run for additional diagnostics: ${TEST_STDERR}"
- diff ${TEST_STDOUT} ${TEST_REF}
- NUM_FAILURES=$(( NUM_FAILURES + 1 ))
- fi
- done
- done
-
- if [ "${SPARK_EXISTS}" = "FALSE" ]; then
- rm -rf ${SPARK_HOME}
- export SPARK_HOME=""
- fi
-done
-
-# clean the output dir
-rm -rf ${SCRIPT_DIR}/output
-
-loginfo "Tests completed with ${NUM_FAILURES} failures"
-if (( ${NUM_FAILURES} > 0 )); then
- exit 1
-else
- exit 0
-fi
diff --git a/plugins/spark/v4.0/regtests/setup.sh b/plugins/spark/v4.0/regtests/setup.sh
deleted file mode 100755
index 50b8ff2dd8..0000000000
--- a/plugins/spark/v4.0/regtests/setup.sh
+++ /dev/null
@@ -1,238 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-###################################
-# Idempotent setup for spark regression tests. Run manually or let run.sh auto-run.
-#
-# Warning - first time setup may download large amounts of files
-# Warning - may clobber conf/spark-defaults.conf
-# Warning - it will set the SPARK_HOME environment variable with the spark setup
-#
-# The script can be called independently like following
-# ./setup.sh --sparkVersion ${SPARK_VERSION} --scalaVersion ${SCALA_VERSION} --jar ${JAR_PATH} --tableFormat ${TABLE_FORMAT}
-# Required Parameters:
-# --sparkVersion : the spark version to setup
-# --scalaVersion : the scala version of spark to setup
-# --jar : path to the local Polaris Spark client jar
-#
-# Optional Parameters:
-# --tableFormat : table format to configure (delta|hudi). Default: delta
-#
-
-set -x
-
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-SPARK_VERSION=3.5.6
-SCALA_VERSION=2.12
-POLARIS_CLIENT_JAR=""
-POLARIS_VERSION=""
-TABLE_FORMAT="delta"
-while [[ $# -gt 0 ]]; do
- case "$1" in
- --sparkVersion)
- SPARK_VERSION="$2"
- shift # past argument
- shift # past value
- ;;
- --scalaVersion)
- SCALA_VERSION="$2"
- shift # past argument
- shift # past value
- ;;
- --polarisVersion)
- POLARIS_VERSION="$2"
- shift # past argument
- shift # past value
- ;;
- --jar)
- POLARIS_CLIENT_JAR="$2"
- shift # past argument
- shift # past value
- ;;
- --tableFormat)
- TABLE_FORMAT="$2"
- shift # past argument
- shift # past value
- ;;
- --) shift;
- break
- ;;
- esac
-done
-
-echo "SET UP FOR SPARK_VERSION=${SPARK_VERSION} SCALA_VERSION=${SCALA_VERSION} POLARIS_VERSION=${POLARIS_VERSION} POLARIS_CLIENT_JAR=${POLARIS_CLIENT_JAR} TABLE_FORMAT=${TABLE_FORMAT}"
-
-# Validate table format
-if [[ "$TABLE_FORMAT" != "delta" && "$TABLE_FORMAT" != "hudi" ]]; then
- echo "Error: Invalid table format '${TABLE_FORMAT}'. Must be 'delta' or 'hudi'."
- exit 1
-fi
-
-if [ "$SCALA_VERSION" == "2.12" ]; then
- SPARK_DISTRIBUTION=spark-${SPARK_VERSION}-bin-hadoop3
-else
- SPARK_DISTRIBUTION=spark-${SPARK_VERSION}-bin-hadoop3-scala${SCALA_VERSION}
-fi
-
-echo "Getting spark distribution ${SPARK_DISTRIBUTION}"
-
-if [ -z "${SPARK_HOME}" ]; then
- SPARK_HOME=$(realpath ~/${SPARK_DISTRIBUTION})
-fi
-SPARK_CONF="${SPARK_HOME}/conf/spark-defaults.conf"
-DERBY_HOME="/tmp/derby"
-
-# Ensure binaries are downloaded locally
-echo 'Verifying Spark binaries...'
-if ! [ -f ${SPARK_HOME}/bin/spark-sql ]; then
- echo 'Setting up Spark...'
- if [ -z "${SPARK_VERSION}" ] || [ -z "${SPARK_DISTRIBUTION}" ]; then
- echo 'SPARK_VERSION or SPARK_DISTRIBUTION not set. Please set SPARK_VERSION and SPARK_DISTRIBUTION to the desired version.'
- exit 1
- fi
- if ! [ -f ~/${SPARK_DISTRIBUTION}.tgz ]; then
- echo 'Downloading spark distro...'
- wget -O ~/${SPARK_DISTRIBUTION}.tgz https://www.apache.org/dyn/closer.lua/spark/spark-${SPARK_VERSION}/${SPARK_DISTRIBUTION}.tgz?action=download
- if ! [ -f ~/${SPARK_DISTRIBUTION}.tgz ]; then
- if [[ "${OSTYPE}" == "darwin"* ]]; then
- echo "Detected OS: mac. Running 'brew install wget' to try again."
- brew install wget
- wget -O ~/${SPARK_DISTRIBUTION}.tgz https://www.apache.org/dyn/closer.lua/spark/spark-${SPARK_VERSION}/${SPARK_DISTRIBUTION}.tgz?action=download
- fi
- fi
- else
- echo 'Found existing Spark tarball'
- fi
- # check if the download was successful
- if ! [ -f ~/${SPARK_DISTRIBUTION}.tgz ]; then
- echo 'Failed to download Spark distribution. Please check the logs.'
- exit 1
- fi
- tar xzvf ~/${SPARK_DISTRIBUTION}.tgz -C ~/${TEST_ROOT_DIR}
- if [ $? -ne 0 ]; then
- echo 'Failed to extract Spark distribution. Please check the logs.'
- exit 1
- else
- echo 'Extracted Spark distribution.'
- rm ~/${SPARK_DISTRIBUTION}.tgz
- fi
- SPARK_HOME=$(realpath ~/${SPARK_DISTRIBUTION})
- SPARK_CONF="${SPARK_HOME}/conf/spark-defaults.conf"
-else
- echo 'Verified Spark distro already installed.'
-fi
-
-echo "SPARK_HOME=${SPARK_HOME}"
-echo "SPARK_CONF=${SPARK_CONF}"
-
-# Ensure Spark boilerplate conf is set
-echo 'Verifying Spark conf...'
-if grep 'POLARIS_TESTCONF_V5' ${SPARK_CONF} 2>/dev/null; then
- echo 'Verified spark conf'
-else
- echo 'Setting spark conf...'
- # Instead of clobbering existing spark conf, just comment it all out in case it was customized carefully.
- sed -i 's/^/# /' ${SPARK_CONF}
-
-# If POLARIS_CLIENT_JAR is provided, set the spark conf to use the jars configuration.
-# Otherwise use the packages setting
-if [[ -z "$POLARIS_CLIENT_JAR" ]]; then
- cat << EOF >> ${SPARK_CONF}
-# POLARIS Spark client test conf
-EOF
- if [[ "$TABLE_FORMAT" == "hudi" ]]; then
- cat << EOF >> ${SPARK_CONF}
-spark.jars.packages org.apache.polaris:polaris-spark-3.5_$SCALA_VERSION:$POLARIS_VERSION,org.apache.hudi:hudi-spark3.5-bundle_${SCALA_VERSION}:1.1.1
-# Note: Hudi package is passed via --packages on command line in spark_sql_hudi.sh
-# to ensure it's resolved before Kryo initialization
-EOF
- else
- cat << EOF >> ${SPARK_CONF}
-spark.jars.packages org.apache.polaris:polaris-spark-3.5_$SCALA_VERSION:$POLARIS_VERSION,io.delta:delta-spark_${SCALA_VERSION}:3.2.1
-EOF
- fi
-else
- cat << EOF >> ${SPARK_CONF}
-# POLARIS Spark client test conf
-spark.jars $POLARIS_CLIENT_JAR
-EOF
- if [[ "$TABLE_FORMAT" == "hudi" ]]; then
- cat << EOF >> ${SPARK_CONF}
-spark.jars.packages org.apache.hudi:hudi-spark3.5-bundle_${SCALA_VERSION}:1.1.1
-EOF
- else
- cat << EOF >> ${SPARK_CONF}
-spark.jars.packages io.delta:delta-spark_${SCALA_VERSION}:3.2.1
-EOF
- fi
-fi
-
-cat << EOF >> ${SPARK_CONF}
-
-spark.sql.variable.substitute true
-
-spark.driver.extraJavaOptions -Dderby.system.home=${DERBY_HOME}
-
-EOF
-
-if [[ "$TABLE_FORMAT" == "hudi" ]]; then
- cat << EOF >> ${SPARK_CONF}
-spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions,org.apache.spark.sql.hudi.HoodieSparkSessionExtension
-# this configuration is needed for hudi table
-spark.sql.catalog.spark_catalog=org.apache.spark.sql.hudi.catalog.HoodieCatalog
-spark.serializer=org.apache.spark.serializer.KryoSerializer
-spark.kryo.registrator=org.apache.spark.HoodieSparkKryoRegistrar
-hoodie.metadata.enable=false
-EOF
-else
- cat << EOF >> ${SPARK_CONF}
-spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions,io.delta.sql.DeltaSparkSessionExtension
-# this configuration is needed for delta table
-spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog
-EOF
-fi
-
-cat << EOF >> ${SPARK_CONF}
-spark.sql.catalog.polaris=org.apache.polaris.spark.SparkCatalog
-spark.sql.catalog.polaris.uri=http://${POLARIS_HOST:-localhost}:8181/api/catalog
-# this configuration is currently only used for iceberg tables, generic tables currently
-# don't support credential vending
-spark.sql.catalog.polaris.header.X-Iceberg-Access-Delegation=vended-credentials
-spark.sql.catalog.polaris.client.region=us-west-2
-# configuration required to ensure DataSourceV2 load works correctly for
-# different table formats
-spark.sql.sources.useV1SourceList=''
-EOF
- echo 'Success!'
-fi
-
-# cleanup derby home if existed
-if [ -d "${DERBY_HOME}" ]; then
- echo "Directory ${DERBY_HOME} exists. Deleting it..."
- rm -rf "${DERBY_HOME}"
-fi
-
-echo "Launch spark-sql at ${SPARK_HOME}/bin/spark-sql"
-# bootstrap dependencies so that future queries don't need to wait for the downloads.
-# this is mostly useful for building the Docker image with all needed dependencies
-${SPARK_HOME}/bin/spark-sql -e "SELECT 1"
-
-# ensure SPARK_HOME is setup for later tests
-export SPARK_HOME=$SPARK_HOME
diff --git a/plugins/spark/v4.0/regtests/suites/spark_sql_delta.ref b/plugins/spark/v4.0/regtests/suites/spark_sql_delta.ref
deleted file mode 100755
index 51698a69ea..0000000000
--- a/plugins/spark/v4.0/regtests/suites/spark_sql_delta.ref
+++ /dev/null
@@ -1,57 +0,0 @@
-{"defaults":{"default-base-location":"file:///tmp/spark_catalog"},"overrides":{"namespace-separator":"%1F","prefix":"spark_sql_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","HEAD /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/ [...]
-Catalog created
-spark-sql (default)> use polaris;
-spark-sql ()> create namespace db1;
-spark-sql ()> create namespace db2;
-spark-sql ()> show namespaces;
-db1
-db2
-spark-sql ()>
- > create namespace db1.schema1;
-spark-sql ()> show namespaces in db1;
-db1.schema1
-spark-sql ()>
- > create table db1.schema1.iceberg_tb (col1 int);
-spark-sql ()> show tables in db1;
-spark-sql ()> show tables in db1.schema1;
-iceberg_tb
-spark-sql ()>
- > use db1.schema1;
-spark-sql (db1.schema1)> insert into iceberg_tb values (123), (234), (111);
-spark-sql (db1.schema1)> select * from iceberg_tb order by col1;
-111
-123
-234
-spark-sql (db1.schema1)>
- > create table delta_tb1(col1 string) using delta location 'file:///tmp/spark_catalog/delta_tb1';
-spark-sql (db1.schema1)> insert into delta_tb1 values ('ab'), ('bb'), ('dd');
-spark-sql (db1.schema1)> select * from delta_tb1 order by col1;
-ab
-bb
-dd
-spark-sql (db1.schema1)>
- > show tables;
-iceberg_tb
-delta_tb1
-spark-sql (db1.schema1)>
- > use db1;
-spark-sql (db1)> create table delta_tb2(col1 int) using delta location 'file:///tmp/spark_catalog/delta_tb2';
-spark-sql (db1)> insert into delta_tb2 values (1), (2), (3) order by col1;
-spark-sql (db1)> select * from delta_tb2;
-1
-2
-3
-spark-sql (db1)>
- > show tables;
-delta_tb2
-spark-sql (db1)> show tables in db1.schema1;
-iceberg_tb
-delta_tb1
-spark-sql (db1)>
- > drop table db1.schema1.iceberg_tb;
-spark-sql (db1)> drop table db1.schema1.delta_tb1;
-spark-sql (db1)> drop namespace db1.schema1;
-spark-sql (db1)> drop table delta_tb2;
-spark-sql (db1)> drop namespace db1;
-spark-sql (db1)> drop namespace db2;
-spark-sql (db1)>
diff --git a/plugins/spark/v4.0/regtests/suites/spark_sql_delta.sh b/plugins/spark/v4.0/regtests/suites/spark_sql_delta.sh
deleted file mode 100755
index fe036664cd..0000000000
--- a/plugins/spark/v4.0/regtests/suites/spark_sql_delta.sh
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN}"
-
-CATALOG_NAME="spark_sql_catalog"
-curl -i -X POST -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \
- http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs \
- -d '{"name": "spark_sql_catalog", "id": 100, "type": "INTERNAL", "readOnly": false, "properties": {"default-base-location": "file:///tmp/spark_catalog"}, "storageConfigInfo": {"storageType": "FILE", "allowedLocations": ["file:///tmp"]}}' > /dev/stderr
-
-# Add TABLE_WRITE_DATA to the catalog's catalog_admin role since by default it can only manage access and metadata
-curl -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \
- http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs/${CATALOG_NAME}/catalog-roles/catalog_admin/grants \
- -d '{"type": "catalog", "privilege": "TABLE_WRITE_DATA"}' > /dev/stderr
-
-curl -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \
- "http://${POLARIS_HOST:-localhost}:8181/api/catalog/v1/config?warehouse=${CATALOG_NAME}"
-echo
-echo "Catalog created"
-cat << EOF | ${SPARK_HOME}/bin/spark-sql -S --conf spark.sql.catalog.polaris.token="${SPARK_BEARER_TOKEN}" --conf spark.sql.catalog.polaris.warehouse=${CATALOG_NAME}
-use polaris;
-create namespace db1;
-create namespace db2;
-show namespaces;
-
-create namespace db1.schema1;
-show namespaces in db1;
-
-create table db1.schema1.iceberg_tb (col1 int);
-show tables in db1;
-show tables in db1.schema1;
-
-use db1.schema1;
-insert into iceberg_tb values (123), (234), (111);
-select * from iceberg_tb order by col1;
-
-create table delta_tb1(col1 string) using delta location 'file:///tmp/spark_catalog/delta_tb1';
-insert into delta_tb1 values ('ab'), ('bb'), ('dd');
-select * from delta_tb1 order by col1;
-
-show tables;
-
-use db1;
-create table delta_tb2(col1 int) using delta location 'file:///tmp/spark_catalog/delta_tb2';
-insert into delta_tb2 values (1), (2), (3) order by col1;
-select * from delta_tb2;
-
-show tables;
-show tables in db1.schema1;
-
-drop table db1.schema1.iceberg_tb;
-drop table db1.schema1.delta_tb1;
-drop namespace db1.schema1;
-drop table delta_tb2;
-drop namespace db1;
-drop namespace db2;
-EOF
-
-# clean up the spark_catalog dir
-rm -rf /tmp/spark_catalog/
-
-curl -i -X DELETE -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \
- http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs/${CATALOG_NAME} > /dev/stderr
diff --git a/plugins/spark/v4.0/regtests/suites/spark_sql_hudi.ref b/plugins/spark/v4.0/regtests/suites/spark_sql_hudi.ref
deleted file mode 100644
index 2889ab613f..0000000000
--- a/plugins/spark/v4.0/regtests/suites/spark_sql_hudi.ref
+++ /dev/null
@@ -1,45 +0,0 @@
-{"defaults":{"default-base-location":"file:///tmp/spark_hudi_catalog"},"overrides":{"namespace-separator":"%1F","prefix":"spark_hudi_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","HEAD /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/t [...]
-Catalog created
-spark-sql (default)> use polaris;
-spark-sql ()> create namespace hudi_db1;
-spark-sql ()> create namespace hudi_db2;
-spark-sql ()> show namespaces;
-hudi_db1
-hudi_db2
-spark-sql ()>
- > create namespace hudi_db1.schema1;
-spark-sql ()> show namespaces in hudi_db1;
-hudi_db1.schema1
-spark-sql ()>
- > create table hudi_db1.schema1.hudi_tb1 (id int, name string) using hudi location 'file:///tmp/spark_hudi_catalog/hudi_tb1';
-spark-sql ()> show tables in hudi_db1;
-spark-sql ()> show tables in hudi_db1.schema1;
-spark-sql ()>
- > use hudi_db1.schema1;
-spark-sql (hudi_db1.schema1)> insert into hudi_tb1 values (1, 'alice'), (2, 'bob');
-spark-sql (hudi_db1.schema1)> select * from hudi_tb1 order by id;
-spark-sql (hudi_db1.schema1)>
- > create table hudi_tb2 (name string, age int, country string) using hudi partitioned by (country) location 'file:///tmp/spark_hudi_catalog/hudi_tb2';
-spark-sql (hudi_db1.schema1)> insert into hudi_tb2 values ('anna', 10, 'US'), ('james', 32, 'US'), ('yan', 16, 'CHINA');
-spark-sql (hudi_db1.schema1)> select name, country from hudi_tb2 order by age;
-spark-sql (hudi_db1.schema1)>
- > show tables;
-spark-sql (hudi_db1.schema1)>
- > use hudi_db1;
-spark-sql (hudi_db1)> create table iceberg_tb (col1 int);
-spark-sql (hudi_db1)> insert into iceberg_tb values (100), (200);
-spark-sql (hudi_db1)> select * from iceberg_tb order by col1;
-100
-200
-spark-sql (hudi_db1)>
- > show tables;
-iceberg_tb
-spark-sql (hudi_db1)> show tables in hudi_db1.schema1;
-spark-sql (hudi_db1)>
- > drop table hudi_db1.schema1.hudi_tb1;
-spark-sql (hudi_db1)> drop table hudi_db1.schema1.hudi_tb2;
-spark-sql (hudi_db1)> drop namespace hudi_db1.schema1;
-spark-sql (hudi_db1)> drop table iceberg_tb;
-spark-sql (hudi_db1)> drop namespace hudi_db1;
-spark-sql (hudi_db1)> drop namespace hudi_db2;
-spark-sql (hudi_db1)>
diff --git a/plugins/spark/v4.0/regtests/suites/spark_sql_hudi.sh b/plugins/spark/v4.0/regtests/suites/spark_sql_hudi.sh
deleted file mode 100755
index 0a230a4d0f..0000000000
--- a/plugins/spark/v4.0/regtests/suites/spark_sql_hudi.sh
+++ /dev/null
@@ -1,85 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN}"
-
-# Determine Scala version (default to 2.12 if not set)
-SCALA_VERSION="${SCALA_VERSION:-2.12}"
-
-CATALOG_NAME="spark_hudi_catalog"
-curl -i -X POST -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \
- http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs \
- -d '{"name": "spark_hudi_catalog", "id": 200, "type": "INTERNAL", "readOnly": false, "properties": {"default-base-location": "file:///tmp/spark_hudi_catalog"}, "storageConfigInfo": {"storageType": "FILE", "allowedLocations": ["file:///tmp"]}}' > /dev/stderr
-
-# Add TABLE_WRITE_DATA to the catalog's catalog_admin role since by default it can only manage access and metadata
-curl -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \
- http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs/${CATALOG_NAME}/catalog-roles/catalog_admin/grants \
- -d '{"type": "catalog", "privilege": "TABLE_WRITE_DATA"}' > /dev/stderr
-
-curl -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \
- "http://${POLARIS_HOST:-localhost}:8181/api/catalog/v1/config?warehouse=${CATALOG_NAME}"
-echo
-echo "Catalog created"
-cat << EOF | ${SPARK_HOME}/bin/spark-sql -S --conf spark.sql.catalog.polaris.token="${SPARK_BEARER_TOKEN}" --conf spark.sql.catalog.polaris.warehouse=${CATALOG_NAME}
-use polaris;
-create namespace hudi_db1;
-create namespace hudi_db2;
-show namespaces;
-
-create namespace hudi_db1.schema1;
-show namespaces in hudi_db1;
-
-create table hudi_db1.schema1.hudi_tb1 (id int, name string) using hudi location 'file:///tmp/spark_hudi_catalog/hudi_tb1';
-show tables in hudi_db1;
-show tables in hudi_db1.schema1;
-
-use hudi_db1.schema1;
-insert into hudi_tb1 values (1, 'alice'), (2, 'bob');
-select * from hudi_tb1 order by id;
-
-create table hudi_tb2 (name string, age int, country string) using hudi partitioned by (country) location 'file:///tmp/spark_hudi_catalog/hudi_tb2';
-insert into hudi_tb2 values ('anna', 10, 'US'), ('james', 32, 'US'), ('yan', 16, 'CHINA');
-select name, country from hudi_tb2 order by age;
-
-show tables;
-
-use hudi_db1;
-create table iceberg_tb (col1 int);
-insert into iceberg_tb values (100), (200);
-select * from iceberg_tb order by col1;
-
-show tables;
-show tables in hudi_db1.schema1;
-
-drop table hudi_db1.schema1.hudi_tb1;
-drop table hudi_db1.schema1.hudi_tb2;
-drop namespace hudi_db1.schema1;
-drop table iceberg_tb;
-drop namespace hudi_db1;
-drop namespace hudi_db2;
-EOF
-
-# clean up the spark_hudi_catalog dir
-rm -rf /tmp/spark_hudi_catalog/
-
-curl -i -X DELETE -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \
- http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs/${CATALOG_NAME} > /dev/stderr
-
diff --git a/plugins/spark/v4.0/spark/BUNDLE-LICENSE b/plugins/spark/v4.0/spark/BUNDLE-LICENSE
index 3f7701abe0..205de6db6f 100644
--- a/plugins/spark/v4.0/spark/BUNDLE-LICENSE
+++ b/plugins/spark/v4.0/spark/BUNDLE-LICENSE
@@ -204,8 +204,8 @@
This product bundles and includes code from Apache Iceberg.
-* plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/PolarisRESTCatalog.java
-* plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/SparkCatalog.java
+* plugins/spark/v4.0/spark/src/main/java/org/apache/polaris/spark/PolarisRESTCatalog.java
+* plugins/spark/v4.0/spark/src/main/java/org/apache/polaris/spark/SparkCatalog.java
Copyright: 2017-2025 The Apache Software Foundation
Project URL: https://iceberg.apache.org/
diff --git a/plugins/spark/v4.0/spark/build.gradle.kts b/plugins/spark/v4.0/spark/build.gradle.kts
index 90bb71e374..c665fa3486 100644
--- a/plugins/spark/v4.0/spark/build.gradle.kts
+++ b/plugins/spark/v4.0/spark/build.gradle.kts
@@ -35,17 +35,12 @@ sourceSets {
}
// get version information
-val sparkMajorVersion = "3.5"
+val sparkMajorVersion = "4.0"
val scalaVersion = getAndUseScalaVersionForProject()
val icebergVersion = libs.versions.iceberg.get()
-val spark35Version = libs.versions.spark35.get()
+val spark40Version = libs.versions.spark40.get()
-val scalaLibraryVersion =
- if (scalaVersion == "2.12") {
- libs.versions.scala212.get()
- } else {
- libs.versions.scala213.get()
- }
+val scalaLibraryVersion = libs.versions.scala213.get()
dependencies {
// TODO: extract a polaris-rest module as a thin layer for
@@ -58,7 +53,7 @@ dependencies {
compileOnly("org.scala-lang:scala-library:${scalaLibraryVersion}")
compileOnly("org.scala-lang:scala-reflect:${scalaLibraryVersion}")
- compileOnly("org.apache.spark:spark-sql_${scalaVersion}:${spark35Version}") {
+ compileOnly("org.apache.spark:spark-sql_${scalaVersion}:${spark40Version}") {
// exclude log4j dependencies
exclude("org.apache.logging.log4j", "log4j-slf4j2-impl")
exclude("org.apache.logging.log4j", "log4j-api")
@@ -78,15 +73,16 @@ dependencies {
testImplementation(libs.mockito.core)
testImplementation(
- "org.apache.iceberg:iceberg-spark-runtime-3.5_${scalaVersion}:${icebergVersion}"
+ "org.apache.iceberg:iceberg-spark-runtime-4.0_${scalaVersion}:${icebergVersion}"
)
- testImplementation("org.apache.spark:spark-sql_${scalaVersion}:${spark35Version}") {
+ testImplementation("org.apache.spark:spark-sql_${scalaVersion}:${spark40Version}") {
// exclude log4j dependencies
exclude("org.apache.logging.log4j", "log4j-slf4j2-impl")
exclude("org.apache.logging.log4j", "log4j-api")
exclude("org.apache.logging.log4j", "log4j-1.2-api")
exclude("org.slf4j", "jul-to-slf4j")
}
+ testImplementation("org.apache.logging.log4j:log4j-api:2.24.3")
}
tasks.register<ShadowJar>("createPolarisSparkJar") {
diff --git a/plugins/spark/v4.0/spark/checkstyle_suppressions.xml b/plugins/spark/v4.0/spark/checkstyle_suppressions.xml
index 1f6f0aad91..c4449f0451 100644
--- a/plugins/spark/v4.0/spark/checkstyle_suppressions.xml
+++ b/plugins/spark/v4.0/spark/checkstyle_suppressions.xml
@@ -23,10 +23,10 @@
<suppressions>
<!--
- Suppress IllegalImport in all files under plugins/spark/v3.5/spark.
+ Suppress IllegalImport in all files under plugins/spark/v4.0/spark and common.
The spark client relies on the shaded libraries from iceberg-spark-runtime,
and therefore uses imports like org.apache.iceberg.shaded.* in the code,
which is intended.
-->
- <suppress checks="IllegalImport" files="plugins/spark/v3.5/spark/.*"/>
-</suppressions>
+ <suppress checks="IllegalImport" files="plugins/spark/(v4.0/spark|common)/.*"/>
+</suppressions>
\ No newline at end of file
diff --git a/plugins/spark/v4.0/spark/src/main/java/org/apache/polaris/spark/SparkCatalog.java b/plugins/spark/v4.0/spark/src/main/java/org/apache/polaris/spark/SparkCatalog.java
index 377a6da0a6..a06a50be98 100644
--- a/plugins/spark/v4.0/spark/src/main/java/org/apache/polaris/spark/SparkCatalog.java
+++ b/plugins/spark/v4.0/spark/src/main/java/org/apache/polaris/spark/SparkCatalog.java
@@ -50,6 +50,7 @@ import org.apache.spark.sql.connector.catalog.TableChange;
import org.apache.spark.sql.connector.catalog.View;
import org.apache.spark.sql.connector.catalog.ViewCatalog;
import org.apache.spark.sql.connector.catalog.ViewChange;
+import org.apache.spark.sql.connector.catalog.ViewInfo;
import org.apache.spark.sql.connector.expressions.Transform;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
@@ -316,27 +317,9 @@ public class SparkCatalog
}
@Override
- public View createView(
- Identifier ident,
- String sql,
- String currentCatalog,
- String[] currentNamespace,
- StructType schema,
- String[] queryColumnNames,
- String[] columnAliases,
- String[] columnComments,
- Map<String, String> properties)
+ public View createView(ViewInfo viewInfo)
throws ViewAlreadyExistsException, NoSuchNamespaceException {
- return this.icebergsSparkCatalog.createView(
- ident,
- sql,
- currentCatalog,
- currentNamespace,
- schema,
- queryColumnNames,
- columnAliases,
- columnComments,
- properties);
+ return this.icebergsSparkCatalog.createView(viewInfo);
}
@Override
@@ -356,6 +339,12 @@ public class SparkCatalog
this.icebergsSparkCatalog.renameView(fromIdentifier, toIdentifier);
}
+ @Override
+ public View replaceView(ViewInfo viewInfo, boolean orCreate)
+ throws NoSuchNamespaceException, NoSuchViewException {
+ return this.icebergsSparkCatalog.replaceView(viewInfo, orCreate);
+ }
+
@Override
public View replaceView(
Identifier ident,
diff --git a/plugins/spark/v4.0/spark/src/main/java/org/apache/polaris/spark/utils/PolarisCatalogUtils.java b/plugins/spark/v4.0/spark/src/main/java/org/apache/polaris/spark/utils/PolarisCatalogUtils.java
index 0eade78658..b431916c1b 100644
--- a/plugins/spark/v4.0/spark/src/main/java/org/apache/polaris/spark/utils/PolarisCatalogUtils.java
+++ b/plugins/spark/v4.0/spark/src/main/java/org/apache/polaris/spark/utils/PolarisCatalogUtils.java
@@ -158,7 +158,7 @@ public class PolarisCatalogUtils {
Option.apply(genericTable.format()),
emptyStringSeq,
scala.Option.empty(),
- genericTable.properties().get(TableCatalog.PROP_OWNER),
+ genericTable.properties().getOrDefault(TableCatalog.PROP_OWNER, ""),
System.currentTimeMillis(),
-1L,
"",
@@ -166,6 +166,7 @@ public class PolarisCatalogUtils {
scala.Option.empty(),
scala.Option.empty(),
scala.Option.empty(),
+ scala.Option.empty(),
emptyStringSeq,
false,
true,
diff --git a/plugins/spark/v4.0/spark/src/test/java/org/apache/polaris/spark/SparkCatalogTest.java b/plugins/spark/v4.0/spark/src/test/java/org/apache/polaris/spark/SparkCatalogTest.java
index 2e4350699b..e65b28b10a 100644
--- a/plugins/spark/v4.0/spark/src/test/java/org/apache/polaris/spark/SparkCatalogTest.java
+++ b/plugins/spark/v4.0/spark/src/test/java/org/apache/polaris/spark/SparkCatalogTest.java
@@ -53,6 +53,7 @@ import org.apache.spark.sql.connector.catalog.TableProvider;
import org.apache.spark.sql.connector.catalog.V1Table;
import org.apache.spark.sql.connector.catalog.View;
import org.apache.spark.sql.connector.catalog.ViewChange;
+import org.apache.spark.sql.connector.catalog.ViewInfo;
import org.apache.spark.sql.connector.expressions.Transform;
import org.apache.spark.sql.execution.datasources.DataSource;
import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Utils;
@@ -139,13 +140,20 @@ public class SparkCatalogTest {
// Setup persistent SparkSession mock
mockedStaticSparkSession = Mockito.mockStatic(SparkSession.class);
mockedSession = Mockito.mock(SparkSession.class);
+ mockedStaticSparkSession.when(SparkSession::active).thenReturn(mockedSession);
+ mockedStaticSparkSession
+ .when(SparkSession::getActiveSession)
+ .thenReturn(scala.Option.apply(mockedSession));
+ mockedStaticSparkSession
+ .when(SparkSession::getDefaultSession)
+ .thenReturn(scala.Option.apply(mockedSession));
+
org.apache.spark.sql.RuntimeConfig mockedConfig =
Mockito.mock(org.apache.spark.sql.RuntimeConfig.class);
SparkContext mockedContext = Mockito.mock(SparkContext.class);
SessionState mockedSessionState = Mockito.mock(SessionState.class);
SQLConf mockedSQLConf = Mockito.mock(SQLConf.class);
- mockedStaticSparkSession.when(SparkSession::active).thenReturn(mockedSession);
Mockito.when(mockedSession.conf()).thenReturn(mockedConfig);
Mockito.when(mockedSession.sessionState()).thenReturn(mockedSessionState);
Mockito.when(mockedSessionState.conf()).thenReturn(mockedSQLConf);
@@ -160,7 +168,7 @@ public class SparkCatalogTest {
Mockito.when(mockedContext.applicationId()).thenReturn("appId");
Mockito.when(mockedContext.appName()).thenReturn("test-app");
Mockito.when(mockedContext.sparkUser()).thenReturn("test-user");
- Mockito.when(mockedContext.version()).thenReturn("3.5");
+ Mockito.when(mockedContext.version()).thenReturn("4.0");
try (MockedStatic<SparkUtil> mockedSparkUtil = Mockito.mockStatic(SparkUtil.class)) {
mockedSparkUtil
@@ -306,15 +314,16 @@ public class SparkCatalogTest {
String viewSql = "select id from test-table where id < 3";
StructType schema = new StructType().add("id", "long");
catalog.createView(
- viewIdentifier,
- viewSql,
- catalogName,
- defaultNS,
- schema,
- new String[0],
- new String[0],
- new String[0],
- Maps.newHashMap());
+ new ViewInfo(
+ viewIdentifier,
+ viewSql,
+ catalogName,
+ defaultNS,
+ schema,
+ new String[0],
+ new String[0],
+ new String[0],
+ Maps.newHashMap()));
// load the view
View view = catalog.loadView(viewIdentifier);
@@ -340,15 +349,17 @@ public class SparkCatalogTest {
Map<String, String> properties = Maps.newHashMap();
properties.put("key1", "value1");
catalog.replaceView(
- newIdentifier,
- newSql,
- catalogName,
- defaultNS,
- schema,
- new String[0],
- new String[0],
- new String[0],
- properties);
+ new ViewInfo(
+ newIdentifier,
+ newSql,
+ catalogName,
+ defaultNS,
+ schema,
+ new String[0],
+ new String[0],
+ new String[0],
+ properties),
+ false);
view = catalog.loadView(newIdentifier);
assertThat(view.query()).isEqualTo(newSql);
assertThat(view.properties()).contains(Map.entry("key1", "value1"));
@@ -368,15 +379,16 @@ public class SparkCatalogTest {
String view1Name = "test-view1";
String view1SQL = "select id from test-table where id >= 3";
catalog.createView(
- Identifier.of(defaultNS, view1Name),
- view1SQL,
- catalogName,
- defaultNS,
- defaultSchema,
- new String[0],
- new String[0],
- new String[0],
- Maps.newHashMap());
+ new ViewInfo(
+ Identifier.of(defaultNS, view1Name),
+ view1SQL,
+ catalogName,
+ defaultNS,
+ defaultSchema,
+ new String[0],
+ new String[0],
+ new String[0],
+ Maps.newHashMap()));
// create two views under ns.nsl2
String[] nsl2ViewNames = new String[] {"test-view2", "test-view3"};
String[] nsl2ViewSQLs =
@@ -385,15 +397,16 @@ public class SparkCatalogTest {
};
for (int i = 0; i < nsl2ViewNames.length; i++) {
catalog.createView(
- Identifier.of(namespace, nsl2ViewNames[i]),
- nsl2ViewSQLs[i],
- catalogName,
- namespace,
- defaultSchema,
- new String[0],
- new String[0],
- new String[0],
- Maps.newHashMap());
+ new ViewInfo(
+ Identifier.of(namespace, nsl2ViewNames[i]),
+ nsl2ViewSQLs[i],
+ catalogName,
+ namespace,
+ defaultSchema,
+ new String[0],
+ new String[0],
+ new String[0],
+ Maps.newHashMap()));
}
// list views under defaultNS
Identifier[] l1Views = catalog.listViews(defaultNS);
diff --git a/settings.gradle.kts b/settings.gradle.kts
index 841c2b1b82..596a03a225 100644
--- a/settings.gradle.kts
+++ b/settings.gradle.kts
@@ -68,7 +68,9 @@ val sparkVersions = sparkScalaVersions["sparkVersions"].toString().split(",").ma
val noSourceChecksProjects = mutableSetOf<String>()
for (sparkVersion in sparkVersions) {
- val scalaVersions = sparkScalaVersions["scalaVersions"].toString().split(",").map { it.trim() }
+ val scalaVersionsKey = "scalaVersions.${sparkVersion}"
+ val scalaVersionsStr = sparkScalaVersions[scalaVersionsKey].toString()
+ val scalaVersions = scalaVersionsStr.split(",").map { it.trim() }
var first = true
for (scalaVersion in scalaVersions) {
val sparkArtifactId = "polaris-spark-${sparkVersion}_${scalaVersion}"