You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by "HyukjinKwon (via GitHub)" <gi...@apache.org> on 2025/02/28 11:22:48 UTC
Re: [PR] [SPARK-50714][SQL][SS] Enable schema evolution for TransformWithState when Avro encoding is used [spark]
HyukjinKwon commented on code in PR #49277:
URL: https://github.com/apache/spark/pull/49277#discussion_r1975249747
##########
python/pyspark/sql/tests/pandas/test_pandas_transform_with_state.py:
##########
@@ -1294,6 +1310,167 @@ def test_transform_with_state_with_timers_single_partition(self):
self.test_transform_with_state_in_pandas_proc_timer()
self.test_transform_with_state_restart_with_multiple_rows_init_state()
+ def _run_evolution_test(
+ self, processor, checkpoint_dir, check_results, df, check_exception=None
+ ):
+ output_schema = StructType(
+ [
+ StructField("id", StringType(), True),
+ StructField("value", processor.state_schema, True),
+ ]
+ )
+
+ # Stop any active streams first
+ for q in self.spark.streams.active:
+ q.stop()
+
+ try:
+ q = (
+ df.groupBy("id")
+ .transformWithStateInPandas(
+ statefulProcessor=processor,
+ outputStructType=output_schema,
+ outputMode="Update",
+ timeMode="None",
+ )
+ .writeStream.queryName("evolution_test")
+ .option("checkpointLocation", checkpoint_dir)
+ .foreachBatch(check_results)
+ .outputMode("update")
+ .start()
+ )
+
+ self.assertEqual(q.name, "evolution_test")
+ self.assertTrue(q.isActive)
+ q.processAllAvailable()
+ q.awaitTermination(10)
+
+ if q.exception() is None:
+ assert check_exception is None
+
+ except Exception as e:
+ # If we are expecting an exception, verify it's the right one
+ if check_exception is None:
+ raise # Re-raise if we weren't expecting an exception
+ self.assertTrue(check_exception(e))
+
+ def test_schema_evolution_scenarios(self):
Review Comment:
This test seems flaky
```
======================================================================
ERROR [40.891s]: test_schema_evolution_scenarios (pyspark.sql.tests.connect.pandas.test_parity_pandas_transform_with_state.TransformWithStateInPandasParityTests.test_schema_evolution_scenarios)
Test various schema evolution scenarios
----------------------------------------------------------------------
Traceback (most recent call last):
File "/__w/spark/spark/python/pyspark/sql/tests/pandas/test_pandas_transform_with_state.py", line 1343, in test_schema_evolution_scenarios
self._run_evolution_test(
File "/__w/spark/spark/python/pyspark/sql/tests/pandas/test_pandas_transform_with_state.py", line 1291, in _run_evolution_test
q.processAllAvailable()
File "/__w/spark/spark/python/pyspark/sql/connect/streaming/query.py", line 137, in processAllAvailable
self._execute_streaming_query_cmd(cmd)
File "/__w/spark/spark/python/pyspark/sql/connect/streaming/query.py", line 185, in _execute_streaming_query_cmd
(_, properties, _) = self._session.client.execute_command(exec_cmd)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/__w/spark/spark/python/pyspark/sql/connect/client/core.py", line 1135, in execute_command
data, _, metrics, observed_metrics, properties = self._execute_and_fetch(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/__w/spark/spark/python/pyspark/sql/connect/client/core.py", line 1547, in _execute_and_fetch
for response in self._execute_and_fetch_as_iterator(
File "/__w/spark/spark/python/pyspark/sql/connect/client/core.py", line 1524, in _execute_and_fetch_as_iterator
self._handle_error(error)
File "/__w/spark/spark/python/pyspark/sql/connect/client/core.py", line 1798, in _handle_error
self._handle_rpc_error(error)
File "/__w/spark/spark/python/pyspark/sql/connect/client/core.py", line 1874, in _handle_rpc_error
raise convert_exception(
pyspark.errors.exceptions.connect.StreamingQueryException: [STREAM_FAILED] Query [id = 7b099714-cfbd-447a-81dd-33a1b812316a, runId = 9773b932-90da-4cf3-b6d4-059b0aa7f530] terminated with exception: [FOREACH_BATCH_USER_FUNCTION_ERROR] An error occurred in the user provided function in foreach batch sink. Reason: [session: 524635cb-b47a-4134-b2ed-058246f08999] [userId: ] Found error inside foreachBatch Python process: Traceback (most recent call last):
File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/sql/connect/streaming/worker/foreach_batch_worker.py", line 84, in main
process(df_ref_id, int(batch_id))
File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/sql/connect/streaming/worker/foreach_batch_worker.py", line 53, in process
func(batch_df, batch_id)
File "/__w/spark/spark/python/pyspark/sql/tests/pandas/test_pandas_transform_with_state.py", line 1340, in check_reorder_fields
assert result.value["name"] == "name-00"
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError
SQLSTATE: 39000 SQLSTATE: XXKST
=== Streaming Query ===
Identifier: evolution_test [id = 7b099714-cfbd-447a-81dd-33a1b812316a, runId = 9773b932-90da-4cf3-b6d4-059b0aa7f530]
Current Committed Offsets: {FileStreamSource[file:/__w/spark/spark/python/target/12d1b762-b507-47a8-b77b-77ae6cd38ae7/tmpy0ktfcpm]: {"logOffset":2}}
Current Available Offsets: {FileStreamSource[file:/__w/spark/spark/python/target/12d1b762-b507-47a8-b77b-77ae6cd38ae7/tmpy0ktfcpm]: {"logOffset":3}}
Current State: ACTIVE
Thread State: RUNNABLE
Logical Plan:
~WriteToMicroBatchDataSourceV1 ForeachBatchSink, 7b099714-cfbd-447a-81dd-33a1b812316a, [checkpointLocation=/__w/spark/spark/python/target/12d1b762-b507-47a8-b77b-77ae6cd38ae7/tmp8vph_mya, queryName=evolution_test], Update
+- ~TransformWithStateInPandas transformWithStateUDF(id#1590, temperature#1591)#1588, 1, [id#1592, value#1593], Update, NoTime, false, 0
:- ~Project [id#1590, id#1590, temperature#1591]
: +- ~Project [cast(split_values#1589[0] as string) AS id#1590, cast(split_values#1589[1] as int) AS temperature#1591]
: +- ~Project [value#822, split(value#822, ,, -1) AS split_values#1589]
: +- ~StreamingExecutionRelation FileStreamSource[file:/__w/spark/spark/python/target/12d1b762-b507-47a8-b77b-77ae6cd38ae7/tmpy0ktfcpm], [value#822]
+- LocalRelation <empty>
JVM stacktrace:
org.apache.spark.sql.streaming.StreamingQueryException
at org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:372)
at org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:226)
Caused by: org.apache.spark.api.python.PythonException: [session: 524635cb-b47a-4134-b2ed-058246f08999] [userId: ] Found error inside foreachBatch Python process: Traceback (most recent call last):
File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/sql/connect/streaming/worker/foreach_batch_worker.py", line 84, in main
process(df_ref_id, int(batch_id))
File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/sql/connect/streaming/worker/foreach_batch_worker.py", line 53, in process
func(batch_df, batch_id)
File "/__w/spark/spark/python/pyspark/sql/tests/pandas/test_pandas_transform_with_state.py", line 1340, in check_reorder_fields
assert result.value["name"] == "name-00"
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError
at org.apache.spark.sql.connect.planner.StreamingForeachBatchHelper$.$anonfun$pythonForeachBatchWrapper$3(StreamingForeachBatchHelper.scala:183)
at org.apache.spark.sql.connect.planner.StreamingForeachBatchHelper$.$anonfun$pythonForeachBatchWrapper$3$adapted(StreamingForeachBatchHelper.scala:156)
at org.apache.spark.sql.connect.planner.StreamingForeachBatchHelper$.$anonfun$dataFrameCachingWrapper$1(StreamingForeachBatchHelper.scala:81)
at org.apache.spark.sql.connect.planner.StreamingForeachBatchHelper$.$anonfun$dataFrameCachingWrapper$1$adapted(StreamingForeachBatchHelper.scala:68)
at org.apache.spark.sql.execution.streaming.sources.ForeachBatchSink.callBatchWriter(ForeachBatchSink.scala:56)
at org.apache.spark.sql.execution.streaming.sources.ForeachBatchSink.addBatch(ForeachBatchSink.scala:49)
at org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runBatch$17(MicroBatchExecution.scala:879)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:162)
at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:268)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:124)
at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:186)
at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:124)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:291)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:123)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:77)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:233)
at org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runBatch$16(MicroBatchExecution.scala:876)
at org.apache.spark.sql.execution.streaming.ProgressContext.reportTimeTaken(ProgressReporter.scala:186)
at org.apache.spark.sql.execution.streaming.MicroBatchExecution.runBatch(MicroBatchExecution.scala:876)
at org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$executeOneBatch$2(MicroBatchExecution.scala:394)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
at org.apache.spark.sql.execution.streaming.ProgressContext.reportTimeTaken(ProgressReporter.scala:186)
at org.apache.spark.sql.execution.streaming.MicroBatchExecution.executeOneBatch(MicroBatchExecution.scala:364)
at org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runActivatedStream$1(MicroBatchExecution.scala:344)
at org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runActivatedStream$1$adapted(MicroBatchExecution.scala:344)
at org.apache.spark.sql.execution.streaming.TriggerExecutor.runOneBatch(TriggerExecutor.scala:39)
at org.apache.spark.sql.execution.streaming.TriggerExecutor.runOneBatch$(TriggerExecutor.scala:37)
at org.apache.spark.sql.execution.streaming.ProcessingTimeExecutor.runOneBatch(TriggerExecutor.scala:70)
at org.apache.spark.sql.execution.streaming.ProcessingTimeExecutor.execute(TriggerExecutor.scala:82)
at org.apache.spark.sql.execution.streaming.MicroBatchExecution.runActivatedStream(MicroBatchExecution.scala:344)
at org.apache.spark.sql.execution.streaming.StreamExecution.$anonfun$runStream$1(StreamExecution.scala:337)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
at org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:311)
at org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:226)
----------------------------------------------------------------------
```
https://github.com/apache/spark/actions/runs/13582276329/job/37970350709
@ericm-db would you mind taking a look please?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org