You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by mb...@apache.org on 2026/04/08 15:57:00 UTC
(datafusion-comet) branch main updated: ci: Remove native_datafusion workflows after 4f5eaf0. (#3908)
This is an automated email from the ASF dual-hosted git repository.
mbutrovich pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 67624a80b ci: Remove native_datafusion workflows after 4f5eaf0. (#3908)
67624a80b is described below
commit 67624a80b8c4829c72679b59bcfb56410a6b7b40
Author: Matt Butrovich <mb...@users.noreply.github.com>
AuthorDate: Wed Apr 8 11:56:56 2026 -0400
ci: Remove native_datafusion workflows after 4f5eaf0. (#3908)
---
.github/workflows/pr_build_linux.yml | 5 --
.github/workflows/spark_sql_test.yml | 10 +--
.../workflows/spark_sql_test_native_datafusion.yml | 72 ----------------------
3 files changed, 2 insertions(+), 85 deletions(-)
diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml
index 35d1c0ac3..030657fc3 100644
--- a/.github/workflows/pr_build_linux.yml
+++ b/.github/workflows/pr_build_linux.yml
@@ -250,11 +250,6 @@ jobs:
maven_opts: "-Pspark-3.5 -Dspark.version=3.5.6 -Pscala-2.13"
scan_impl: "auto"
- - name: "Spark 3.5, JDK 17, Scala 2.12"
- java_version: "17"
- maven_opts: "-Pspark-3.5 -Pscala-2.12"
- scan_impl: "native_datafusion"
-
- name: "Spark 3.5, JDK 17, Scala 2.12"
java_version: "17"
maven_opts: "-Pspark-3.5 -Pscala-2.12"
diff --git a/.github/workflows/spark_sql_test.yml b/.github/workflows/spark_sql_test.yml
index 8f48078ca..61e36a006 100644
--- a/.github/workflows/spark_sql_test.yml
+++ b/.github/workflows/spark_sql_test.yml
@@ -121,22 +121,16 @@ jobs:
- {name: "sql_hive-1", args1: "", args2: "hive/testOnly * -- -l org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest"}
- {name: "sql_hive-2", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.ExtendedHiveTest"}
- {name: "sql_hive-3", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.SlowHiveTest"}
- # Test combinations:
- # - auto scan: all Spark versions (3.4, 3.5, 4.0)
- # - native_iceberg_compat: Spark 3.5 only
+ # Since 4f5eaf0, auto mode uses native_datafusion for V1 scans,
+ # so we only need to test with auto.
config:
- {spark-short: '3.4', spark-full: '3.4.3', java: 11, scan-impl: 'auto'}
- - {spark-short: '3.4', spark-full: '3.4.3', java: 11, scan-impl: 'native_datafusion'}
- {spark-short: '3.5', spark-full: '3.5.8', java: 11, scan-impl: 'auto'}
- - {spark-short: '3.5', spark-full: '3.5.8', java: 11, scan-impl: 'native_datafusion'}
- {spark-short: '4.0', spark-full: '4.0.1', java: 17, scan-impl: 'auto'}
- - {spark-short: '4.0', spark-full: '4.0.1', java: 17, scan-impl: 'native_datafusion'}
# Skip sql_hive-1 for Spark 4.0 due to https://github.com/apache/datafusion-comet/issues/2946
exclude:
- config: {spark-short: '4.0', spark-full: '4.0.1', java: 17, scan-impl: 'auto'}
module: {name: "sql_hive-1", args1: "", args2: "hive/testOnly * -- -l org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest"}
- - config: {spark-short: '4.0', spark-full: '4.0.1', java: 17, scan-impl: 'native_datafusion'}
- module: {name: "sql_hive-1", args1: "", args2: "hive/testOnly * -- -l org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest"}
fail-fast: false
name: spark-sql-${{ matrix.config.scan-impl }}-${{ matrix.module.name }}/spark-${{ matrix.config.spark-full }}
runs-on: ${{ matrix.os }}
diff --git a/.github/workflows/spark_sql_test_native_datafusion.yml b/.github/workflows/spark_sql_test_native_datafusion.yml
deleted file mode 100644
index 8df9d4801..000000000
--- a/.github/workflows/spark_sql_test_native_datafusion.yml
+++ /dev/null
@@ -1,72 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Spark SQL Tests (native_datafusion)
-
-concurrency:
- group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
- cancel-in-progress: true
-
-on:
- # manual trigger
- # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
- workflow_dispatch:
-
-env:
- RUST_VERSION: stable
- RUST_BACKTRACE: 1
-
-jobs:
- spark-sql-catalyst-native-datafusion:
- strategy:
- matrix:
- os: [ubuntu-24.04]
- java-version: [11]
- spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.8'}]
- module:
- - {name: "catalyst", args1: "catalyst/test", args2: ""}
- - {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}
- - {name: "sql/core-2", args1: "", args2: "sql/testOnly * -- -n org.apache.spark.tags.ExtendedSQLTest"}
- - {name: "sql/core-3", args1: "", args2: "sql/testOnly * -- -n org.apache.spark.tags.SlowSQLTest"}
- - {name: "sql/hive-1", args1: "", args2: "hive/testOnly * -- -l org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest"}
- - {name: "sql/hive-2", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.ExtendedHiveTest"}
- - {name: "sql/hive-3", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.SlowHiveTest"}
- fail-fast: false
- name: spark-sql-native-datafusion-${{ matrix.module.name }}/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }}
- runs-on: ${{ matrix.os }}
- container:
- image: amd64/rust
- steps:
- - uses: actions/checkout@v6
- - name: Setup Rust & Java toolchain
- uses: ./.github/actions/setup-builder
- with:
- rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{ matrix.java-version }}
- - name: Setup Spark
- uses: ./.github/actions/setup-spark-builder
- with:
- spark-version: ${{ matrix.spark-version.full }}
- spark-short-version: ${{ matrix.spark-version.short }}
- - name: Run Spark tests
- run: |
- cd apache-spark
- rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
- ENABLE_COMET=true ENABLE_COMET_ONHEAP=true COMET_PARQUET_SCAN_IMPL=native_datafusion build/sbt -Dsbt.log.noformat=true ${{ matrix.module.args1 }} "${{ matrix.module.args2 }}"
- env:
- LC_ALL: "C.UTF-8"
-
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org