You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2025/11/19 04:28:43 UTC
(spark) branch master updated: [SPARK-54375][CONNECT][TESTS][FOLLOWUP] Make `PythonPipelineSuite` perform a default check for `PythonTestDepsChecker.isConnectDepsAvailable`
This is an automated email from the ASF dual-hosted git repository.
yangjie01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1aa8d5aaa11d [SPARK-54375][CONNECT][TESTS][FOLLOWUP] Make `PythonPipelineSuite` perform a default check for `PythonTestDepsChecker.isConnectDepsAvailable`
1aa8d5aaa11d is described below
commit 1aa8d5aaa11d062f57aa0184e1b75d23cf974930
Author: yangjie01 <ya...@baidu.com>
AuthorDate: Wed Nov 19 12:28:14 2025 +0800
[SPARK-54375][CONNECT][TESTS][FOLLOWUP] Make `PythonPipelineSuite` perform a default check for `PythonTestDepsChecker.isConnectDepsAvailable`
### What changes were proposed in this pull request?
This pr aims to make the test cases in `PythonPipelineSuite` perform a default check for `PythonTestDepsChecker.isConnectDepsAvailable`.
### Why are the changes needed?
Simplify the dependency checks for Python modules in test cases within `PythonPipelineSuite`.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Pass Github Actions
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #53106 from LuciferYang/refactor-PythonPipelineSuite.
Authored-by: yangjie01 <ya...@baidu.com>
Signed-off-by: yangjie01 <ya...@baidu.com>
---
.../connect/pipelines/PythonPipelineSuite.scala | 22 ++++++++++++----------
1 file changed, 12 insertions(+), 10 deletions(-)
diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/pipelines/PythonPipelineSuite.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/pipelines/PythonPipelineSuite.scala
index 826e2338589d..98b33c3296fa 100644
--- a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/pipelines/PythonPipelineSuite.scala
+++ b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/pipelines/PythonPipelineSuite.scala
@@ -26,6 +26,9 @@ import java.util.concurrent.TimeUnit
import scala.collection.mutable.ArrayBuffer
import scala.util.Try
+import org.scalactic.source.Position
+import org.scalatest.Tag
+
import org.apache.spark.api.python.PythonUtils
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.TableIdentifier
@@ -49,7 +52,6 @@ class PythonPipelineSuite
with EventVerificationTestHelpers {
def buildGraph(pythonText: String): DataflowGraph = {
- assume(PythonTestDepsChecker.isConnectDepsAvailable)
val indentedPythonText = pythonText.linesIterator.map(" " + _).mkString("\n")
// create a unique identifier to allow identifying the session and dataflow graph
val customSessionIdentifier = UUID.randomUUID().toString
@@ -530,7 +532,6 @@ class PythonPipelineSuite
"eager analysis or execution will fail")(
Seq("""spark.sql("SELECT * FROM src")""", """spark.read.table("src").collect()""")) {
command =>
- assume(PythonTestDepsChecker.isConnectDepsAvailable)
val ex = intercept[RuntimeException] {
buildGraph(s"""
|@dp.materialized_view
@@ -549,7 +550,6 @@ class PythonPipelineSuite
}
test("create dataset with the same name will fail") {
- assume(PythonTestDepsChecker.isConnectDepsAvailable)
val ex = intercept[AnalysisException] {
buildGraph(s"""
|@dp.materialized_view
@@ -623,7 +623,6 @@ class PythonPipelineSuite
}
test("create datasets with three part names") {
- assume(PythonTestDepsChecker.isConnectDepsAvailable)
val graphTry = Try {
buildGraph(s"""
|@dp.table(name = "some_catalog.some_schema.mv")
@@ -676,7 +675,6 @@ class PythonPipelineSuite
}
test("create named flow with multipart name will fail") {
- assume(PythonTestDepsChecker.isConnectDepsAvailable)
val ex = intercept[RuntimeException] {
buildGraph(s"""
|@dp.table
@@ -825,7 +823,6 @@ class PythonPipelineSuite
}
test("create pipeline without table will throw RUN_EMPTY_PIPELINE exception") {
- assume(PythonTestDepsChecker.isConnectDepsAvailable)
checkError(
exception = intercept[AnalysisException] {
buildGraph(s"""
@@ -837,7 +834,6 @@ class PythonPipelineSuite
}
test("create pipeline with only temp view will throw RUN_EMPTY_PIPELINE exception") {
- assume(PythonTestDepsChecker.isConnectDepsAvailable)
checkError(
exception = intercept[AnalysisException] {
buildGraph(s"""
@@ -851,7 +847,6 @@ class PythonPipelineSuite
}
test("create pipeline with only flow will throw RUN_EMPTY_PIPELINE exception") {
- assume(PythonTestDepsChecker.isConnectDepsAvailable)
checkError(
exception = intercept[AnalysisException] {
buildGraph(s"""
@@ -1048,7 +1043,6 @@ class PythonPipelineSuite
gridTest("Unsupported SQL command outside query function should result in a failure")(
unsupportedSqlCommandList) { unsupportedSqlCommand =>
- assume(PythonTestDepsChecker.isConnectDepsAvailable)
val ex = intercept[RuntimeException] {
buildGraph(s"""
|spark.sql("$unsupportedSqlCommand")
@@ -1063,7 +1057,6 @@ class PythonPipelineSuite
gridTest("Unsupported SQL command inside query function should result in a failure")(
unsupportedSqlCommandList) { unsupportedSqlCommand =>
- assume(PythonTestDepsChecker.isConnectDepsAvailable)
val ex = intercept[RuntimeException] {
buildGraph(s"""
|@dp.materialized_view()
@@ -1111,4 +1104,13 @@ class PythonPipelineSuite
| return spark.range(5)
|""".stripMargin)
}
+
+ override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit
+ pos: Position): Unit = {
+ if (PythonTestDepsChecker.isConnectDepsAvailable) {
+ super.test(testName, testTags: _*)(testFun)
+ } else {
+ super.ignore(testName, testTags: _*)(testFun)
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org