You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ha...@apache.org on 2025/02/24 09:58:01 UTC
(spark) branch master updated: [SPARK-51278][PYTHON] Use appropriate structure of JSON format for `PySparkLogger`
This is an automated email from the ASF dual-hosted git repository.
haejoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 67a337e202af [SPARK-51278][PYTHON] Use appropriate structure of JSON format for `PySparkLogger`
67a337e202af is described below
commit 67a337e202afcd4f32a0fd593488fbad21751f76
Author: Haejoon Lee <ha...@databricks.com>
AuthorDate: Mon Feb 24 18:57:48 2025 +0900
[SPARK-51278][PYTHON] Use appropriate structure of JSON format for `PySparkLogger`
### What changes were proposed in this pull request?
This PR proposes to use appropriate structure of JSON format for `PySparkLogger` by updating `JSONFormatter`.
### Why are the changes needed?
To align the log structure with JVM side. Currently the "stacktrace" does not align.
### Does this PR introduce _any_ user-facing change?
The output JSON structure will include "stacktrace" in correct format:
**Before**
```json
{
"ts": "2025-02-21 15:40:52,043",
"level": "ERROR",
"logger": "TestLogger",
"msg": "Exception occurred",
"context": {"user": "test_user_stacktrace"},
"exception": {
"class": "ValueError",
"msg": "Test Exception",
"stacktrace": [
"Traceback (most recent call last):", " File \"/spark/python/pyspark/logger/tests/test_logger.py\", line 115, in test_log_exception_with_stacktrace", " raise ValueError(\"Test Exception\")", "ValueError: Test Exception"
]
}
}
```
**After**
```json
{
"ts": "2025-02-21 15:40:52,043",
"level": "ERROR",
"logger": "TestLogger",
"msg": "Exception occurred",
"context": {"user": "test_user_stacktrace"},
"exception": {
"class": "ValueError",
"msg": "Test Exception",
"stacktrace": [
{
"class": null,
"method": "test_log_exception_with_stacktrace",
"file": "/spark/python/pyspark/logger/tests/test_logger.py",
"line": "115"
}
]
}
}
```
### How was this patch tested?
Added related UT
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #50038 from itholic/fix_json_format.
Lead-authored-by: Haejoon Lee <ha...@databricks.com>
Co-authored-by: Hyukjin Kwon <gu...@gmail.com>
Signed-off-by: Haejoon Lee <ha...@databricks.com>
---
python/pyspark/logger/logger.py | 16 ++++++++++++++--
python/pyspark/logger/tests/test_logger.py | 29 ++++++++++++++++++++++++++++-
2 files changed, 42 insertions(+), 3 deletions(-)
diff --git a/python/pyspark/logger/logger.py b/python/pyspark/logger/logger.py
index 8485a4205fb6..36667106c5ea 100644
--- a/python/pyspark/logger/logger.py
+++ b/python/pyspark/logger/logger.py
@@ -18,6 +18,7 @@
import logging
import json
+import traceback
import sys
from typing import cast, Mapping, Optional, TYPE_CHECKING
@@ -30,7 +31,7 @@ SPARK_LOG_SCHEMA = (
"msg STRING, "
"context map<STRING, STRING>, "
"exception STRUCT<class STRING, msg STRING, "
- "stacktrace ARRAY<STRUCT<class STRING, method STRING, file STRING,line STRING>>>,"
+ "stacktrace ARRAY<STRUCT<class STRING, method STRING, file STRING, line STRING>>>, "
"logger STRING"
)
@@ -70,10 +71,21 @@ class JSONFormatter(logging.Formatter):
}
if record.exc_info:
exc_type, exc_value, exc_tb = record.exc_info
+ stacktrace = traceback.extract_tb(exc_tb)
+
+ structured_stacktrace = [
+ {
+ "class": None,
+ "method": frame.name,
+ "file": frame.filename,
+ "line": str(frame.lineno),
+ }
+ for frame in stacktrace
+ ]
log_entry["exception"] = {
"class": exc_type.__name__ if exc_type else "UnknownException",
"msg": str(exc_value),
- "stacktrace": self.formatException(record.exc_info).splitlines(),
+ "stacktrace": structured_stacktrace,
}
return json.dumps(log_entry, ensure_ascii=False)
diff --git a/python/pyspark/logger/tests/test_logger.py b/python/pyspark/logger/tests/test_logger.py
index cf29cdbd0512..52250a494f52 100644
--- a/python/pyspark/logger/tests/test_logger.py
+++ b/python/pyspark/logger/tests/test_logger.py
@@ -124,7 +124,34 @@ class LoggerTestsMixin:
self.assertTrue(
log_json["exception"]["class"] in ("Py4JJavaError", "_MultiThreadedRendezvous")
)
- self.assertTrue("Traceback" in log_json["exception"]["stacktrace"][0])
+ stacktrace = log_json["exception"]["stacktrace"][0].keys()
+ self.assertTrue("class" in stacktrace)
+ self.assertTrue("method" in stacktrace)
+ self.assertTrue("file" in stacktrace)
+ self.assertTrue("line" in stacktrace)
+
+ def test_log_exception_with_stacktrace(self):
+ try:
+ raise ValueError("Test Exception")
+ except ValueError:
+ self.logger.exception("Exception occurred", user="test_user_stacktrace")
+
+ log_json = json.loads(self.handler.stream.getvalue().strip())
+
+ self.assertEqual(log_json["msg"], "Exception occurred")
+ self.assertEqual(log_json["context"], {"user": "test_user_stacktrace"})
+ self.assertTrue("exception" in log_json)
+ self.assertTrue("class" in log_json["exception"])
+ self.assertTrue("msg" in log_json["exception"])
+ self.assertTrue("stacktrace" in log_json["exception"])
+ self.assertIsInstance(log_json["exception"]["stacktrace"], list)
+
+ # Check the structure of "stacktrace"
+ for frame in log_json["exception"]["stacktrace"]:
+ self.assertTrue("class" in frame)
+ self.assertTrue("method" in frame)
+ self.assertTrue("file" in frame)
+ self.assertTrue("line" in frame)
class LoggerTests(LoggerTestsMixin, ReusedSQLTestCase):
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org