You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2026/03/09 04:10:25 UTC
(paimon-rust) branch main updated: ci: introduce spark paimon integration for integration test (#107)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 115e79c ci: introduce spark paimon integration for integration test (#107)
115e79c is described below
commit 115e79ce0da913dd5db760e687864d19987d8ec9
Author: yuxia Luo <lu...@alumni.sjtu.edu.cn>
AuthorDate: Mon Mar 9 12:10:20 2026 +0800
ci: introduce spark paimon integration for integration test (#107)
---
.github/workflows/ci.yml | 13 +++++++++++
Makefile | 38 +++++++++++++++++++++++++++++++
dev/docker-compose.yaml | 41 +++++++++++++++++++++++++++++++++
dev/spark/Dockerfile | 53 +++++++++++++++++++++++++++++++++++++++++++
dev/spark/provision.py | 42 ++++++++++++++++++++++++++++++++++
dev/spark/spark-defaults.conf | 25 ++++++++++++++++++++
6 files changed, 212 insertions(+)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6f25f99..50d727e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -67,8 +67,21 @@ jobs:
- windows-latest
steps:
- uses: actions/checkout@v4
+
- name: Test
run: cargo test --all-targets --workspace
env:
RUST_LOG: DEBUG
RUST_BACKTRACE: full
+
+ integration:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Start Docker containers
+ run: make docker-up
+
+ - name: Stop Docker containers
+ if: always()
+ run: make docker-down
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..1e18aa7
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Version defaults (can be overridden via environment variables)
+PAIMON_VERSION ?= 1.3.1
+
+# Warehouse path for integration tests
+PAIMON_WAREHOUSE ?= /tmp/paimon-warehouse
+
+.EXPORT_ALL_VARIABLES:
+
+# Docker targets for integration tests.
+# Build and run Spark + Paimon container to provision test data.
+# Usage:
+# make docker-up # use default versions
+# PAIMON_VERSION=1.3.0 make docker-up # override Paimon version
+docker-up:
+ docker compose -f dev/docker-compose.yaml build \
+ --build-arg PAIMON_VERSION=$(PAIMON_VERSION)
+ docker compose -f dev/docker-compose.yaml run --rm spark-paimon
+
+# Stop and remove Docker containers, networks, and volumes.
+docker-down:
+ docker compose -f dev/docker-compose.yaml down -v
diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml
new file mode 100644
index 0000000..c5431c0
--- /dev/null
+++ b/dev/docker-compose.yaml
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Spark + Paimon for integration tests: writes tables to a local volume
+# so paimon-rust can read them (no MinIO/S3).
+#
+# Usage:
+# 1. Generate test data (run once):
+# docker compose -f dev/docker-compose.yaml run --rm spark-paimon
+# 2. Run read integration tests on the host:
+# PAIMON_TEST_WAREHOUSE=/tmp/paimon-warehouse cargo test -p paimon-integration-tests read_from_spark
+#
+# To re-provision: remove /tmp/paimon-warehouse and run step 1 again.
+
+services:
+ spark-paimon:
+ build:
+ context: ./spark
+ dockerfile: Dockerfile
+ volumes:
+ # Bind warehouse to host so paimon-rust can read it
+ - /tmp/paimon-warehouse:/tmp/paimon-warehouse
+ # Run as root to avoid permission issues with mounted volumes
+ user: root
+ # Run provision script and exit (data remains in /tmp/paimon-warehouse).
+ # Use full path: entrypoint exec does not have SPARK_HOME/bin in PATH.
+ command: ["/opt/spark/bin/spark-submit", "--master", "local[*]", "/opt/provision.py"]
diff --git a/dev/spark/Dockerfile b/dev/spark/Dockerfile
new file mode 100644
index 0000000..baf0d2b
--- /dev/null
+++ b/dev/spark/Dockerfile
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Spark with Paimon connector for integration tests.
+# Writes Paimon tables to /tmp/paimon-warehouse (mount this as a volume
+# so paimon-rust can read the data).
+#
+# Build args can be overridden via Makefile or docker build:
+# --build-arg PAIMON_VERSION=1.3.1
+FROM apache/spark:3.5.3
+
+# Re-declare ARGs so they are available in this stage (ARGs before FROM are not).
+ARG PAIMON_VERSION
+
+USER root
+WORKDIR ${SPARK_HOME}
+
+RUN apt-get install -y --no-install-recommends curl && \
+ rm -rf /var/lib/apt/lists/*
+
+# Paimon Spark runtime JAR (Spark 3.5)
+# https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.5/1.3.1/paimon-spark-3.5-1.3.1.jar
+ENV PAIMON_JAR="paimon-spark-3.5-${PAIMON_VERSION}.jar"
+ENV PAIMON_JAR_URL="https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.5/${PAIMON_VERSION}/${PAIMON_JAR}"
+
+RUN curl -fsSL --retry 3 -o "${SPARK_HOME}/jars/${PAIMON_JAR}" \
+ "${PAIMON_JAR_URL}" && \
+ chown -R spark:spark "${SPARK_HOME}/jars"
+
+COPY spark-defaults.conf ${SPARK_HOME}/conf/
+COPY provision.py /opt/provision.py
+RUN chown spark:spark ${SPARK_HOME}/conf/spark-defaults.conf /opt/provision.py
+
+USER spark
+
+# Default: run provision script to create tables and write data to /tmp/paimon-warehouse.
+# Override in docker-compose or when running manually.
+# Use full path: entrypoint exec does not have SPARK_HOME/bin in PATH.
+CMD ["/opt/spark/bin/spark-submit", "--master", "local[*]", "/opt/provision.py"]
diff --git a/dev/spark/provision.py b/dev/spark/provision.py
new file mode 100644
index 0000000..00e3cca
--- /dev/null
+++ b/dev/spark/provision.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Provisions Paimon tables into the warehouse (file:/tmp/paimon-warehouse)
+# for paimon-rust integration tests to read.
+
+from pyspark.sql import SparkSession
+
+def main():
+ spark = SparkSession.builder.getOrCreate()
+
+ # Use Paimon catalog (configured in spark-defaults.conf with warehouse file:/tmp/paimon-warehouse)
+ spark.sql("USE paimon.default")
+
+ # Table: simple keyed table for read tests
+ spark.sql("""
+ CREATE TABLE IF NOT EXISTS simple (
+ id INT,
+ name STRING
+ ) USING paimon
+ TBLPROPERTIES ('primary-key' = 'id')
+ """)
+ spark.sql("INSERT INTO simple VALUES (1, 'alice'), (2, 'bob'), (3, 'carol')")
+
+if __name__ == "__main__":
+ main()
diff --git a/dev/spark/spark-defaults.conf b/dev/spark/spark-defaults.conf
new file mode 100644
index 0000000..8f31485
--- /dev/null
+++ b/dev/spark/spark-defaults.conf
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Paimon filesystem catalog – warehouse is mounted at /tmp/paimon-warehouse
+spark.sql.catalog.paimon org.apache.paimon.spark.SparkCatalog
+spark.sql.catalog.paimon.warehouse file:/tmp/paimon-warehouse
+spark.sql.extensions org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions
+
+spark.sql.defaultCatalog paimon