You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ue...@apache.org on 2025/02/08 01:27:17 UTC
(spark) branch branch-4.0 updated: [SPARK-51127][PYTHON] Kill the Python worker on idle timeout
This is an automated email from the ASF dual-hosted git repository.
ueshin pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new a3158c3ff1b6 [SPARK-51127][PYTHON] Kill the Python worker on idle timeout
a3158c3ff1b6 is described below
commit a3158c3ff1b6fdbb4a88653db20cdbbb5e1b9a57
Author: Takuya Ueshin <ue...@databricks.com>
AuthorDate: Fri Feb 7 17:26:56 2025 -0800
[SPARK-51127][PYTHON] Kill the Python worker on idle timeout
### What changes were proposed in this pull request?
Kills the Python worker on idle timeout introduced at #49818.
- Spark conf: `spark.python.worker.killOnIdleTimeout` (default `false`)
Whether Spark should terminate the Python worker process when the idle timeout (as defined by spark.python.worker.idleTimeoutSeconds) is reached. If enabled, Spark will terminate the Python worker process in addition to logging the status.
- SQL conf: `spark.sql.execution.pyspark.udf.killOnIdleTimeout`
The same as `spark.python.worker.killOnIdleTimeout`, but this is a runtime conf for Python UDFs. Falls back to `spark.python.worker.killOnIdleTimeout`.
### Why are the changes needed?
When the Python worker gets stuck, currently there is no way to resume the execution.
### Does this PR introduce _any_ user-facing change?
When `spark.python.worker.idleTimeoutSeconds` is set to a positive number, and set `spark.python.worker.killOnIdleTimeout` to `true`, once it detects a Python worker as stuck, the executor will try to kill the process, which will cause a task failure but Spark will retry the task.
### How was this patch tested?
Added the related tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #49843 from ueshin/issues/SPARK-51127/kill_on_timeout.
Authored-by: Takuya Ueshin <ue...@databricks.com>
Signed-off-by: Takuya Ueshin <ue...@databricks.com>
(cherry picked from commit a5e905d138b2ac85c20696dd5ca553d5cbf15174)
Signed-off-by: Takuya Ueshin <ue...@databricks.com>
---
.../org/apache/spark/api/python/PythonRunner.scala | 33 ++++++++++++++----
.../org/apache/spark/internal/config/Python.scala | 12 ++++++-
python/pyspark/sql/tests/test_udf.py | 17 +++++++++
python/pyspark/sql/tests/test_udtf.py | 40 ++++++++++++++++++++++
.../org/apache/spark/sql/internal/SQLConf.scala | 10 ++++++
.../ApplyInPandasWithStatePythonRunner.scala | 1 +
.../sql/execution/python/ArrowPythonRunner.scala | 1 +
.../execution/python/ArrowPythonUDTFRunner.scala | 1 +
.../python/CoGroupedArrowPythonRunner.scala | 1 +
.../sql/execution/python/PythonForeachWriter.scala | 1 +
.../sql/execution/python/PythonPlannerRunner.scala | 34 ++++++++++++++----
.../sql/execution/python/PythonUDFRunner.scala | 1 +
12 files changed, 139 insertions(+), 13 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
index 771555f67018..752ad8b75946 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
@@ -156,6 +156,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
private val reuseWorker = conf.get(PYTHON_WORKER_REUSE)
protected val faultHandlerEnabled: Boolean = conf.get(PYTHON_WORKER_FAULTHANLDER_ENABLED)
protected val idleTimeoutSeconds: Long = conf.get(PYTHON_WORKER_IDLE_TIMEOUT_SECONDS)
+ protected val killOnIdleTimeout: Boolean = conf.get(PYTHON_WORKER_KILL_ON_IDLE_TIMEOUT)
protected val hideTraceback: Boolean = false
protected val simplifiedTraceback: Boolean = false
@@ -291,7 +292,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
// Return an iterator that read lines from the process's stdout
val dataIn = new DataInputStream(new BufferedInputStream(
- new ReaderInputStream(worker, writer, handle, idleTimeoutSeconds),
+ new ReaderInputStream(worker, writer, handle, idleTimeoutSeconds, killOnIdleTimeout),
bufferSize))
val stdoutIterator = newReaderIterator(
dataIn, writer, startTime, env, worker, handle.map(_.pid.toInt), releasedOrClosed, context)
@@ -679,7 +680,8 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
worker: PythonWorker,
writer: Writer,
handle: Option[ProcessHandle],
- idleTimeoutSeconds: Long) extends InputStream {
+ idleTimeoutSeconds: Long,
+ killOnIdleTimeout: Boolean) extends InputStream {
private[this] var writerIfbhThreadLocalValue: Object = null
private[this] val temp = new Array[Byte](1)
private[this] val bufferStream = new DirectByteBufferOutputStream()
@@ -705,6 +707,8 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
}
}
+ private[this] var pythonWorkerKilled: Boolean = false
+
override def read(b: Array[Byte], off: Int, len: Int): Int = {
// The code below manipulates the InputFileBlockHolder thread local in order
// to prevent behavior changes in the input_file_name() expression due to the switch from
@@ -740,10 +744,27 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
while (n == 0) {
val selected = worker.selector.select(TimeUnit.SECONDS.toMillis(idleTimeoutSeconds))
if (selected == 0) {
- logWarning(log"Idle timeout reached for Python worker (timeout: " +
- log"${MDC(LogKeys.PYTHON_WORKER_IDLE_TIMEOUT, idleTimeoutSeconds)} seconds). " +
- log"No data received from the worker process: " +
- pythonWorkerStatusMessageWithContext(handle, worker, hasInput || buffer.hasRemaining))
+ if (pythonWorkerKilled) {
+ logWarning(
+ log"Waiting for Python worker process to terminate after idle timeout: " +
+ pythonWorkerStatusMessageWithContext(handle, worker, hasInput || buffer.hasRemaining))
+ } else {
+ logWarning(
+ log"Idle timeout reached for Python worker (timeout: " +
+ log"${MDC(LogKeys.PYTHON_WORKER_IDLE_TIMEOUT, idleTimeoutSeconds)} seconds). " +
+ log"No data received from the worker process: " +
+ pythonWorkerStatusMessageWithContext(handle, worker, hasInput || buffer.hasRemaining))
+ if (killOnIdleTimeout) {
+ handle.foreach { handle =>
+ if (handle.isAlive) {
+ logWarning(
+ log"Terminating Python worker process due to idle timeout (timeout: " +
+ log"${MDC(LogKeys.PYTHON_WORKER_IDLE_TIMEOUT, idleTimeoutSeconds)} seconds)")
+ pythonWorkerKilled = handle.destroy()
+ }
+ }
+ }
+ }
}
if (worker.selectionKey.isReadable) {
n = worker.channel.read(buf)
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Python.scala b/core/src/main/scala/org/apache/spark/internal/config/Python.scala
index ed8183a81da4..5622dea50e2e 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Python.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Python.scala
@@ -71,16 +71,26 @@ private[spark] object Python {
.createWithDefault(false)
private val PYTHON_WORKER_IDLE_TIMEOUT_SECONDS_KEY = "spark.python.worker.idleTimeoutSeconds"
+ private val PYTHON_WORKER_KILL_ON_IDLE_TIMEOUT_KEY = "spark.python.worker.killOnIdleTimeout"
val PYTHON_WORKER_IDLE_TIMEOUT_SECONDS = ConfigBuilder(PYTHON_WORKER_IDLE_TIMEOUT_SECONDS_KEY)
.doc("The time (in seconds) Spark will wait for activity " +
"(e.g., data transfer or communication) from a Python worker before considering it " +
"potentially idle or unresponsive. When the timeout is triggered, " +
"Spark will log the network-related status for debugging purposes. " +
- "However, the Python worker will remain active and continue waiting for communication. " +
+ "However, the Python worker will remain active and continue waiting for communication " +
+ s"unless explicitly terminated via $PYTHON_WORKER_KILL_ON_IDLE_TIMEOUT_KEY." +
"The default is `0` that means no timeout.")
.version("4.0.0")
.timeConf(TimeUnit.SECONDS)
.checkValue(_ >= 0, "The idle timeout should be 0 or positive.")
.createWithDefault(0)
+
+ val PYTHON_WORKER_KILL_ON_IDLE_TIMEOUT = ConfigBuilder(PYTHON_WORKER_KILL_ON_IDLE_TIMEOUT_KEY)
+ .doc("Whether Spark should terminate the Python worker process when the idle timeout " +
+ s"(as defined by $PYTHON_WORKER_IDLE_TIMEOUT_SECONDS_KEY) is reached. If enabled, " +
+ "Spark will terminate the Python worker process in addition to logging the status.")
+ .version("4.0.0")
+ .booleanConf
+ .createWithDefault(false)
}
diff --git a/python/pyspark/sql/tests/test_udf.py b/python/pyspark/sql/tests/test_udf.py
index 67d243cd2924..d36f94717773 100644
--- a/python/pyspark/sql/tests/test_udf.py
+++ b/python/pyspark/sql/tests/test_udf.py
@@ -23,6 +23,7 @@ import tempfile
import unittest
import datetime
import io
+import time
from contextlib import redirect_stdout
from pyspark.sql import SparkSession, Column, Row
@@ -1190,6 +1191,22 @@ class BaseUDFTestsMixin(object):
self.spark.range(1).select(udf(lambda x: ctypes.string_at(0))("id")).collect()
+ def test_udf_kill_on_timeout(self):
+ with self.sql_conf(
+ {
+ "spark.sql.execution.pyspark.udf.idleTimeoutSeconds": "1s",
+ "spark.sql.execution.pyspark.udf.killOnIdleTimeout": "true",
+ }
+ ):
+
+ @udf
+ def f(x):
+ time.sleep(2)
+ return str(x)
+
+ with self.assertRaisesRegex(Exception, "Python worker exited unexpectedly"):
+ self.spark.range(1).select(f("id")).show()
+
def test_err_udf_init(self):
with self.quiet():
self.check_err_udf_init()
diff --git a/python/pyspark/sql/tests/test_udtf.py b/python/pyspark/sql/tests/test_udtf.py
index 95a81236fbf5..c727671dd59e 100644
--- a/python/pyspark/sql/tests/test_udtf.py
+++ b/python/pyspark/sql/tests/test_udtf.py
@@ -19,6 +19,7 @@ import platform
import shutil
import tempfile
import unittest
+import time
from dataclasses import dataclass
from typing import Iterator, Optional
@@ -2802,6 +2803,45 @@ class BaseUDTFTestsMixin:
TestUDTFWithAnalyze, None, expected, err_type=Exception
)
+ def test_udtf_kill_on_timeout(self):
+ with self.sql_conf(
+ {
+ "spark.sql.execution.pyspark.udf.idleTimeoutSeconds": "1s",
+ "spark.sql.execution.pyspark.udf.killOnIdleTimeout": "true",
+ }
+ ):
+ with self.subTest(method="eval"):
+
+ class TestUDTF:
+ def eval(self):
+ time.sleep(2)
+ yield "x"
+
+ self._check_result_or_exception(
+ TestUDTF,
+ "x: string",
+ "Python worker exited unexpectedly",
+ err_type=Exception,
+ )
+
+ with self.subTest(method="analyze"):
+
+ class TestUDTFWithAnalyze:
+ @staticmethod
+ def analyze():
+ time.sleep(2)
+ return AnalyzeResult(StructType().add("x", StringType()))
+
+ def eval(self):
+ yield "x",
+
+ self._check_result_or_exception(
+ TestUDTFWithAnalyze,
+ None,
+ "Python worker exited unexpectedly",
+ err_type=Exception,
+ )
+
class UDTFTests(BaseUDTFTestsMixin, ReusedSQLTestCase):
@classmethod
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index b874ec917ee9..c7de22fd4a95 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -3368,6 +3368,14 @@ object SQLConf {
.version("4.0.0")
.fallbackConf(Python.PYTHON_WORKER_IDLE_TIMEOUT_SECONDS)
+ val PYTHON_UDF_WORKER_KILL_ON_IDLE_TIMEOUT =
+ buildConf("spark.sql.execution.pyspark.udf.killOnIdleTimeout")
+ .doc(
+ s"Same as ${Python.PYTHON_WORKER_KILL_ON_IDLE_TIMEOUT.key} for Python execution with " +
+ "DataFrame and SQL. It can change during runtime.")
+ .version("4.0.0")
+ .fallbackConf(Python.PYTHON_WORKER_KILL_ON_IDLE_TIMEOUT)
+
val PYSPARK_PLOT_MAX_ROWS =
buildConf("spark.sql.pyspark.plotting.max_rows")
.doc("The visual limit on plots. If set to 1000 for top-n-based plots (pie, bar, barh), " +
@@ -6282,6 +6290,8 @@ class SQLConf extends Serializable with Logging with SqlApiConf {
def pythonUDFWorkerIdleTimeoutSeconds: Long = getConf(PYTHON_UDF_WORKER_IDLE_TIMEOUT_SECONDS)
+ def pythonUDFWorkerKillOnIdleTimeout: Boolean = getConf(PYTHON_UDF_WORKER_KILL_ON_IDLE_TIMEOUT)
+
def pythonUDFArrowConcurrencyLevel: Option[Int] = getConf(PYTHON_UDF_ARROW_CONCURRENCY_LEVEL)
def pysparkPlotMaxRows: Int = getConf(PYSPARK_PLOT_MAX_ROWS)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ApplyInPandasWithStatePythonRunner.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ApplyInPandasWithStatePythonRunner.scala
index 4ad4e763cffa..8832ac1ea5cb 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ApplyInPandasWithStatePythonRunner.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ApplyInPandasWithStatePythonRunner.scala
@@ -76,6 +76,7 @@ class ApplyInPandasWithStatePythonRunner(
override val faultHandlerEnabled: Boolean = SQLConf.get.pythonUDFWorkerFaulthandlerEnabled
override val idleTimeoutSeconds: Long = SQLConf.get.pythonUDFWorkerIdleTimeoutSeconds
+ override val killOnIdleTimeout: Boolean = SQLConf.get.pythonUDFWorkerKillOnIdleTimeout
private val sqlConf = SQLConf.get
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
index 3da9f0d5a276..d7f73c648ac2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
@@ -48,6 +48,7 @@ abstract class BaseArrowPythonRunner(
override val faultHandlerEnabled: Boolean = SQLConf.get.pythonUDFWorkerFaulthandlerEnabled
override val idleTimeoutSeconds: Long = SQLConf.get.pythonUDFWorkerIdleTimeoutSeconds
+ override val killOnIdleTimeout: Boolean = SQLConf.get.pythonUDFWorkerKillOnIdleTimeout
override val errorOnDuplicatedFieldNames: Boolean = true
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonUDTFRunner.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonUDTFRunner.scala
index 168919287409..0f46f57ddf70 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonUDTFRunner.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonUDTFRunner.scala
@@ -57,6 +57,7 @@ class ArrowPythonUDTFRunner(
override val faultHandlerEnabled: Boolean = SQLConf.get.pythonUDFWorkerFaulthandlerEnabled
override val idleTimeoutSeconds: Long = SQLConf.get.pythonUDFWorkerIdleTimeoutSeconds
+ override val killOnIdleTimeout: Boolean = SQLConf.get.pythonUDFWorkerKillOnIdleTimeout
override val errorOnDuplicatedFieldNames: Boolean = true
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/CoGroupedArrowPythonRunner.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/CoGroupedArrowPythonRunner.scala
index fba24af18726..27d6f7dc1c66 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/CoGroupedArrowPythonRunner.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/CoGroupedArrowPythonRunner.scala
@@ -61,6 +61,7 @@ class CoGroupedArrowPythonRunner(
override val faultHandlerEnabled: Boolean = SQLConf.get.pythonUDFWorkerFaulthandlerEnabled
override val idleTimeoutSeconds: Long = SQLConf.get.pythonUDFWorkerIdleTimeoutSeconds
+ override val killOnIdleTimeout: Boolean = SQLConf.get.pythonUDFWorkerKillOnIdleTimeout
override val hideTraceback: Boolean = SQLConf.get.pysparkHideTraceback
override val simplifiedTraceback: Boolean = SQLConf.get.pysparkSimplifiedTraceback
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonForeachWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonForeachWriter.scala
index fc7386fdce0c..076bc72c431f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonForeachWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonForeachWriter.scala
@@ -100,6 +100,7 @@ class PythonForeachWriter(func: PythonFunction, schema: StructType)
override val faultHandlerEnabled: Boolean = SQLConf.get.pythonUDFWorkerFaulthandlerEnabled
override val idleTimeoutSeconds: Long = SQLConf.get.pythonUDFWorkerIdleTimeoutSeconds
+ override val killOnIdleTimeout: Boolean = SQLConf.get.pythonUDFWorkerKillOnIdleTimeout
override val hideTraceback: Boolean = SQLConf.get.pysparkHideTraceback
override val simplifiedTraceback: Boolean = SQLConf.get.pysparkSimplifiedTraceback
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonPlannerRunner.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonPlannerRunner.scala
index 69cffb0f2116..2b8fa64696c9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonPlannerRunner.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonPlannerRunner.scala
@@ -56,6 +56,7 @@ abstract class PythonPlannerRunner[T](func: PythonFunction) extends Logging {
val localdir = env.blockManager.diskBlockManager.localDirs.map(f => f.getPath()).mkString(",")
val faultHandlerEnabled: Boolean = SQLConf.get.pythonUDFWorkerFaulthandlerEnabled
val idleTimeoutSeconds: Long = SQLConf.get.pythonUDFWorkerIdleTimeoutSeconds
+ val killOnIdleTimeout: Boolean = SQLConf.get.pythonUDFWorkerKillOnIdleTimeout
val hideTraceback: Boolean = SQLConf.get.pysparkHideTraceback
val simplifiedTraceback: Boolean = SQLConf.get.pysparkSimplifiedTraceback
val workerMemoryMb = SQLConf.get.pythonPlannerExecMemory
@@ -112,7 +113,8 @@ abstract class PythonPlannerRunner[T](func: PythonFunction) extends Logging {
dataOut.flush()
val dataIn = new DataInputStream(new BufferedInputStream(
- new WorkerInputStream(worker, bufferStream.toByteBuffer, handle, idleTimeoutSeconds),
+ new WorkerInputStream(
+ worker, bufferStream.toByteBuffer, handle, idleTimeoutSeconds, killOnIdleTimeout),
bufferSize))
val res = receiveFromPython(dataIn)
@@ -170,7 +172,8 @@ abstract class PythonPlannerRunner[T](func: PythonFunction) extends Logging {
worker: PythonWorker,
buffer: ByteBuffer,
handle: Option[ProcessHandle],
- idleTimeoutSeconds: Long) extends InputStream {
+ idleTimeoutSeconds: Long,
+ killOnIdleTimeout: Boolean) extends InputStream {
private[this] val temp = new Array[Byte](1)
@@ -184,16 +187,35 @@ abstract class PythonPlannerRunner[T](func: PythonFunction) extends Logging {
}
}
+ private[this] var pythonWorkerKilled: Boolean = false
+
override def read(b: Array[Byte], off: Int, len: Int): Int = {
val buf = ByteBuffer.wrap(b, off, len)
var n = 0
while (n == 0) {
val selected = worker.selector.select(TimeUnit.SECONDS.toMillis(idleTimeoutSeconds))
if (selected == 0) {
- logWarning(log"Idle timeout reached for Python planner worker (timeout: " +
- log"${MDC(LogKeys.PYTHON_WORKER_IDLE_TIMEOUT, idleTimeoutSeconds)} seconds). " +
- log"No data received from the worker process: " +
- pythonWorkerStatusMessageWithContext(handle, worker, buffer.hasRemaining))
+ if (pythonWorkerKilled) {
+ logWarning(
+ log"Waiting for Python planner worker process to terminate after idle timeout: " +
+ pythonWorkerStatusMessageWithContext(handle, worker, buffer.hasRemaining))
+ } else {
+ logWarning(
+ log"Idle timeout reached for Python planner worker (timeout: " +
+ log"${MDC(LogKeys.PYTHON_WORKER_IDLE_TIMEOUT, idleTimeoutSeconds)} seconds). " +
+ log"No data received from the worker process: " +
+ pythonWorkerStatusMessageWithContext(handle, worker, buffer.hasRemaining))
+ if (killOnIdleTimeout) {
+ handle.foreach { handle =>
+ if (handle.isAlive) {
+ logWarning(
+ log"Terminating Python planner worker process due to idle timeout (timeout: " +
+ log"${MDC(LogKeys.PYTHON_WORKER_IDLE_TIMEOUT, idleTimeoutSeconds)} seconds)")
+ pythonWorkerKilled = handle.destroy()
+ }
+ }
+ }
+ }
}
if (worker.selectionKey.isReadable) {
n = worker.channel.read(buf)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
index 6b5053680637..11773f92c482 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
@@ -47,6 +47,7 @@ abstract class BasePythonUDFRunner(
override val faultHandlerEnabled: Boolean = SQLConf.get.pythonUDFWorkerFaulthandlerEnabled
override val idleTimeoutSeconds: Long = SQLConf.get.pythonUDFWorkerIdleTimeoutSeconds
+ override val killOnIdleTimeout: Boolean = SQLConf.get.pythonUDFWorkerKillOnIdleTimeout
override val bufferSize: Int = SQLConf.get.getConf(SQLConf.PYTHON_UDF_BUFFER_SIZE)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org