You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by "HyukjinKwon (via GitHub)" <gi...@apache.org> on 2025/02/28 11:22:48 UTC

Re: [PR] [SPARK-50714][SQL][SS] Enable schema evolution for TransformWithState when Avro encoding is used [spark]

HyukjinKwon commented on code in PR #49277:
URL: https://github.com/apache/spark/pull/49277#discussion_r1975249747


##########
python/pyspark/sql/tests/pandas/test_pandas_transform_with_state.py:
##########
@@ -1294,6 +1310,167 @@ def test_transform_with_state_with_timers_single_partition(self):
             self.test_transform_with_state_in_pandas_proc_timer()
             self.test_transform_with_state_restart_with_multiple_rows_init_state()
 
+    def _run_evolution_test(
+        self, processor, checkpoint_dir, check_results, df, check_exception=None
+    ):
+        output_schema = StructType(
+            [
+                StructField("id", StringType(), True),
+                StructField("value", processor.state_schema, True),
+            ]
+        )
+
+        # Stop any active streams first
+        for q in self.spark.streams.active:
+            q.stop()
+
+        try:
+            q = (
+                df.groupBy("id")
+                .transformWithStateInPandas(
+                    statefulProcessor=processor,
+                    outputStructType=output_schema,
+                    outputMode="Update",
+                    timeMode="None",
+                )
+                .writeStream.queryName("evolution_test")
+                .option("checkpointLocation", checkpoint_dir)
+                .foreachBatch(check_results)
+                .outputMode("update")
+                .start()
+            )
+
+            self.assertEqual(q.name, "evolution_test")
+            self.assertTrue(q.isActive)
+            q.processAllAvailable()
+            q.awaitTermination(10)
+
+            if q.exception() is None:
+                assert check_exception is None
+
+        except Exception as e:
+            # If we are expecting an exception, verify it's the right one
+            if check_exception is None:
+                raise  # Re-raise if we weren't expecting an exception
+            self.assertTrue(check_exception(e))
+
+    def test_schema_evolution_scenarios(self):

Review Comment:
   This test seems flaky
   
   ```
   ======================================================================
   ERROR [40.891s]: test_schema_evolution_scenarios (pyspark.sql.tests.connect.pandas.test_parity_pandas_transform_with_state.TransformWithStateInPandasParityTests.test_schema_evolution_scenarios)
   Test various schema evolution scenarios
   ----------------------------------------------------------------------
   Traceback (most recent call last):
     File "/__w/spark/spark/python/pyspark/sql/tests/pandas/test_pandas_transform_with_state.py", line 1343, in test_schema_evolution_scenarios
       self._run_evolution_test(
     File "/__w/spark/spark/python/pyspark/sql/tests/pandas/test_pandas_transform_with_state.py", line 1291, in _run_evolution_test
       q.processAllAvailable()
     File "/__w/spark/spark/python/pyspark/sql/connect/streaming/query.py", line 137, in processAllAvailable
       self._execute_streaming_query_cmd(cmd)
     File "/__w/spark/spark/python/pyspark/sql/connect/streaming/query.py", line 185, in _execute_streaming_query_cmd
       (_, properties, _) = self._session.client.execute_command(exec_cmd)
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
     File "/__w/spark/spark/python/pyspark/sql/connect/client/core.py", line 1135, in execute_command
       data, _, metrics, observed_metrics, properties = self._execute_and_fetch(
                                                        ^^^^^^^^^^^^^^^^^^^^^^^^
     File "/__w/spark/spark/python/pyspark/sql/connect/client/core.py", line 1547, in _execute_and_fetch
       for response in self._execute_and_fetch_as_iterator(
     File "/__w/spark/spark/python/pyspark/sql/connect/client/core.py", line 1524, in _execute_and_fetch_as_iterator
       self._handle_error(error)
     File "/__w/spark/spark/python/pyspark/sql/connect/client/core.py", line 1798, in _handle_error
       self._handle_rpc_error(error)
     File "/__w/spark/spark/python/pyspark/sql/connect/client/core.py", line 1874, in _handle_rpc_error
       raise convert_exception(
   pyspark.errors.exceptions.connect.StreamingQueryException: [STREAM_FAILED] Query [id = 7b099714-cfbd-447a-81dd-33a1b812316a, runId = 9773b932-90da-4cf3-b6d4-059b0aa7f530] terminated with exception: [FOREACH_BATCH_USER_FUNCTION_ERROR] An error occurred in the user provided function in foreach batch sink. Reason: [session: 524635cb-b47a-4134-b2ed-058246f08999] [userId: ] Found error inside foreachBatch Python process: Traceback (most recent call last):
     File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/sql/connect/streaming/worker/foreach_batch_worker.py", line 84, in main
       process(df_ref_id, int(batch_id))
     File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/sql/connect/streaming/worker/foreach_batch_worker.py", line 53, in process
       func(batch_df, batch_id)
     File "/__w/spark/spark/python/pyspark/sql/tests/pandas/test_pandas_transform_with_state.py", line 1340, in check_reorder_fields
       assert result.value["name"] == "name-00"
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   AssertionError
    SQLSTATE: 39000 SQLSTATE: XXKST
   === Streaming Query ===
   Identifier: evolution_test [id = 7b099714-cfbd-447a-81dd-33a1b812316a, runId = 9773b932-90da-4cf3-b6d4-059b0aa7f530]
   Current Committed Offsets: {FileStreamSource[file:/__w/spark/spark/python/target/12d1b762-b507-47a8-b77b-77ae6cd38ae7/tmpy0ktfcpm]: {"logOffset":2}}
   Current Available Offsets: {FileStreamSource[file:/__w/spark/spark/python/target/12d1b762-b507-47a8-b77b-77ae6cd38ae7/tmpy0ktfcpm]: {"logOffset":3}}
   
   Current State: ACTIVE
   Thread State: RUNNABLE
   
   Logical Plan:
   ~WriteToMicroBatchDataSourceV1 ForeachBatchSink, 7b099714-cfbd-447a-81dd-33a1b812316a, [checkpointLocation=/__w/spark/spark/python/target/12d1b762-b507-47a8-b77b-77ae6cd38ae7/tmp8vph_mya, queryName=evolution_test], Update
   +- ~TransformWithStateInPandas transformWithStateUDF(id#1590, temperature#1591)#1588, 1, [id#1592, value#1593], Update, NoTime, false, 0
      :- ~Project [id#1590, id#1590, temperature#1591]
      :  +- ~Project [cast(split_values#1589[0] as string) AS id#1590, cast(split_values#1589[1] as int) AS temperature#1591]
      :     +- ~Project [value#822, split(value#822, ,, -1) AS split_values#1589]
      :        +- ~StreamingExecutionRelation FileStreamSource[file:/__w/spark/spark/python/target/12d1b762-b507-47a8-b77b-77ae6cd38ae7/tmpy0ktfcpm], [value#822]
      +- LocalRelation <empty>
   
   
   JVM stacktrace:
   org.apache.spark.sql.streaming.StreamingQueryException
   	at org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:372)
   	at org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:226)
   Caused by: org.apache.spark.api.python.PythonException: [session: 524635cb-b47a-4134-b2ed-058246f08999] [userId: ] Found error inside foreachBatch Python process: Traceback (most recent call last):
     File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/sql/connect/streaming/worker/foreach_batch_worker.py", line 84, in main
       process(df_ref_id, int(batch_id))
     File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/sql/connect/streaming/worker/foreach_batch_worker.py", line 53, in process
       func(batch_df, batch_id)
     File "/__w/spark/spark/python/pyspark/sql/tests/pandas/test_pandas_transform_with_state.py", line 1340, in check_reorder_fields
       assert result.value["name"] == "name-00"
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   AssertionError
   
   	at org.apache.spark.sql.connect.planner.StreamingForeachBatchHelper$.$anonfun$pythonForeachBatchWrapper$3(StreamingForeachBatchHelper.scala:183)
   	at org.apache.spark.sql.connect.planner.StreamingForeachBatchHelper$.$anonfun$pythonForeachBatchWrapper$3$adapted(StreamingForeachBatchHelper.scala:156)
   	at org.apache.spark.sql.connect.planner.StreamingForeachBatchHelper$.$anonfun$dataFrameCachingWrapper$1(StreamingForeachBatchHelper.scala:81)
   	at org.apache.spark.sql.connect.planner.StreamingForeachBatchHelper$.$anonfun$dataFrameCachingWrapper$1$adapted(StreamingForeachBatchHelper.scala:68)
   	at org.apache.spark.sql.execution.streaming.sources.ForeachBatchSink.callBatchWriter(ForeachBatchSink.scala:56)
   	at org.apache.spark.sql.execution.streaming.sources.ForeachBatchSink.addBatch(ForeachBatchSink.scala:49)
   	at org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runBatch$17(MicroBatchExecution.scala:879)
   	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:162)
   	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:268)
   	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:124)
   	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
   	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
   	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:186)
   	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
   	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
   	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:124)
   	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:291)
   	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:123)
   	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
   	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:77)
   	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:233)
   	at org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runBatch$16(MicroBatchExecution.scala:876)
   	at org.apache.spark.sql.execution.streaming.ProgressContext.reportTimeTaken(ProgressReporter.scala:186)
   	at org.apache.spark.sql.execution.streaming.MicroBatchExecution.runBatch(MicroBatchExecution.scala:876)
   	at org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$executeOneBatch$2(MicroBatchExecution.scala:394)
   	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
   	at org.apache.spark.sql.execution.streaming.ProgressContext.reportTimeTaken(ProgressReporter.scala:186)
   	at org.apache.spark.sql.execution.streaming.MicroBatchExecution.executeOneBatch(MicroBatchExecution.scala:364)
   	at org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runActivatedStream$1(MicroBatchExecution.scala:344)
   	at org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runActivatedStream$1$adapted(MicroBatchExecution.scala:344)
   	at org.apache.spark.sql.execution.streaming.TriggerExecutor.runOneBatch(TriggerExecutor.scala:39)
   	at org.apache.spark.sql.execution.streaming.TriggerExecutor.runOneBatch$(TriggerExecutor.scala:37)
   	at org.apache.spark.sql.execution.streaming.ProcessingTimeExecutor.runOneBatch(TriggerExecutor.scala:70)
   	at org.apache.spark.sql.execution.streaming.ProcessingTimeExecutor.execute(TriggerExecutor.scala:82)
   	at org.apache.spark.sql.execution.streaming.MicroBatchExecution.runActivatedStream(MicroBatchExecution.scala:344)
   	at org.apache.spark.sql.execution.streaming.StreamExecution.$anonfun$runStream$1(StreamExecution.scala:337)
   	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
   	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
   	at org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:311)
   	at org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:226)
   
   ----------------------------------------------------------------------
   ```
   
   https://github.com/apache/spark/actions/runs/13582276329/job/37970350709
   
   @ericm-db would you mind taking a look please?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org