You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ha...@apache.org on 2025/02/24 09:58:01 UTC

(spark) branch master updated: [SPARK-51278][PYTHON] Use appropriate structure of JSON format for `PySparkLogger`

This is an automated email from the ASF dual-hosted git repository.

haejoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 67a337e202af [SPARK-51278][PYTHON] Use appropriate structure of JSON format for `PySparkLogger`
67a337e202af is described below

commit 67a337e202afcd4f32a0fd593488fbad21751f76
Author: Haejoon Lee <ha...@databricks.com>
AuthorDate: Mon Feb 24 18:57:48 2025 +0900

    [SPARK-51278][PYTHON] Use appropriate structure of JSON format for `PySparkLogger`
    
    ### What changes were proposed in this pull request?
    
    This PR proposes to use appropriate structure of JSON format for `PySparkLogger` by updating `JSONFormatter`.
    
    ### Why are the changes needed?
    
    To align the log structure with JVM side. Currently the "stacktrace" does not align.
    
    ### Does this PR introduce _any_ user-facing change?
    
    The output JSON structure will include "stacktrace" in correct format:
    
    **Before**
    ```json
    {
      "ts": "2025-02-21 15:40:52,043",
      "level": "ERROR",
      "logger": "TestLogger",
      "msg": "Exception occurred",
      "context": {"user": "test_user_stacktrace"},
      "exception": {
        "class": "ValueError",
        "msg": "Test Exception",
        "stacktrace": [
          "Traceback (most recent call last):", "  File \"/spark/python/pyspark/logger/tests/test_logger.py\", line 115, in test_log_exception_with_stacktrace", "    raise ValueError(\"Test Exception\")", "ValueError: Test Exception"
        ]
      }
    }
    ```
    
    **After**
    ```json
    {
      "ts": "2025-02-21 15:40:52,043",
      "level": "ERROR",
      "logger": "TestLogger",
      "msg": "Exception occurred",
      "context": {"user": "test_user_stacktrace"},
      "exception": {
        "class": "ValueError",
        "msg": "Test Exception",
        "stacktrace": [
          {
            "class": null,
            "method": "test_log_exception_with_stacktrace",
            "file": "/spark/python/pyspark/logger/tests/test_logger.py",
            "line": "115"
          }
        ]
      }
    }
    ```
    
    ### How was this patch tested?
    
    Added related UT
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No
    
    Closes #50038 from itholic/fix_json_format.
    
    Lead-authored-by: Haejoon Lee <ha...@databricks.com>
    Co-authored-by: Hyukjin Kwon <gu...@gmail.com>
    Signed-off-by: Haejoon Lee <ha...@databricks.com>
---
 python/pyspark/logger/logger.py            | 16 ++++++++++++++--
 python/pyspark/logger/tests/test_logger.py | 29 ++++++++++++++++++++++++++++-
 2 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/python/pyspark/logger/logger.py b/python/pyspark/logger/logger.py
index 8485a4205fb6..36667106c5ea 100644
--- a/python/pyspark/logger/logger.py
+++ b/python/pyspark/logger/logger.py
@@ -18,6 +18,7 @@
 
 import logging
 import json
+import traceback
 import sys
 from typing import cast, Mapping, Optional, TYPE_CHECKING
 
@@ -30,7 +31,7 @@ SPARK_LOG_SCHEMA = (
     "msg STRING, "
     "context map<STRING, STRING>, "
     "exception STRUCT<class STRING, msg STRING, "
-    "stacktrace ARRAY<STRUCT<class STRING, method STRING, file STRING,line STRING>>>,"
+    "stacktrace ARRAY<STRUCT<class STRING, method STRING, file STRING, line STRING>>>, "
     "logger STRING"
 )
 
@@ -70,10 +71,21 @@ class JSONFormatter(logging.Formatter):
         }
         if record.exc_info:
             exc_type, exc_value, exc_tb = record.exc_info
+            stacktrace = traceback.extract_tb(exc_tb)
+
+            structured_stacktrace = [
+                {
+                    "class": None,
+                    "method": frame.name,
+                    "file": frame.filename,
+                    "line": str(frame.lineno),
+                }
+                for frame in stacktrace
+            ]
             log_entry["exception"] = {
                 "class": exc_type.__name__ if exc_type else "UnknownException",
                 "msg": str(exc_value),
-                "stacktrace": self.formatException(record.exc_info).splitlines(),
+                "stacktrace": structured_stacktrace,
             }
         return json.dumps(log_entry, ensure_ascii=False)
 
diff --git a/python/pyspark/logger/tests/test_logger.py b/python/pyspark/logger/tests/test_logger.py
index cf29cdbd0512..52250a494f52 100644
--- a/python/pyspark/logger/tests/test_logger.py
+++ b/python/pyspark/logger/tests/test_logger.py
@@ -124,7 +124,34 @@ class LoggerTestsMixin:
             self.assertTrue(
                 log_json["exception"]["class"] in ("Py4JJavaError", "_MultiThreadedRendezvous")
             )
-            self.assertTrue("Traceback" in log_json["exception"]["stacktrace"][0])
+            stacktrace = log_json["exception"]["stacktrace"][0].keys()
+            self.assertTrue("class" in stacktrace)
+            self.assertTrue("method" in stacktrace)
+            self.assertTrue("file" in stacktrace)
+            self.assertTrue("line" in stacktrace)
+
+    def test_log_exception_with_stacktrace(self):
+        try:
+            raise ValueError("Test Exception")
+        except ValueError:
+            self.logger.exception("Exception occurred", user="test_user_stacktrace")
+
+        log_json = json.loads(self.handler.stream.getvalue().strip())
+
+        self.assertEqual(log_json["msg"], "Exception occurred")
+        self.assertEqual(log_json["context"], {"user": "test_user_stacktrace"})
+        self.assertTrue("exception" in log_json)
+        self.assertTrue("class" in log_json["exception"])
+        self.assertTrue("msg" in log_json["exception"])
+        self.assertTrue("stacktrace" in log_json["exception"])
+        self.assertIsInstance(log_json["exception"]["stacktrace"], list)
+
+        # Check the structure of "stacktrace"
+        for frame in log_json["exception"]["stacktrace"]:
+            self.assertTrue("class" in frame)
+            self.assertTrue("method" in frame)
+            self.assertTrue("file" in frame)
+            self.assertTrue("line" in frame)
 
 
 class LoggerTests(LoggerTestsMixin, ReusedSQLTestCase):


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org