You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ge...@apache.org on 2024/10/21 17:42:25 UTC
(spark) branch master updated: [SPARK-50036][CORE][PYTHON] Include SPARK_LOG_SCHEMA in the context of REPL shell
This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new f86df1e37b47 [SPARK-50036][CORE][PYTHON] Include SPARK_LOG_SCHEMA in the context of REPL shell
f86df1e37b47 is described below
commit f86df1e37b4770bffddafc0216ae5eb4b3ac25b7
Author: Gengliang Wang <ge...@apache.org>
AuthorDate: Mon Oct 21 10:42:13 2024 -0700
[SPARK-50036][CORE][PYTHON] Include SPARK_LOG_SCHEMA in the context of REPL shell
### What changes were proposed in this pull request?
Before the Change:
Users needed to import LOG_SCHEMA to read structured logs as a JSON data source:
```
import org.apache.spark.util.LogUtils.LOG_SCHEMA
val logDf = spark.read.schema(LOG_SCHEMA).json("path/to/logs")
```
After the Change:
- Renamed for Clarity:`LOG_SCHEMA` has been renamed to `SPARK_LOG_SCHEMA` to make its purpose more clear.
- No Import Needed in REPL Shells: You can now use `SPARK_LOG_SCHEMA` directly in REPL environments like spark-shell and pyspark without importing it.
Now, you can read structured logs without the import:
```
val logDf = spark.read.schema(SPARK_LOG_SCHEMA).json("path/to/logs")
```
### Why are the changes needed?
Simply the way to query structured logs
### Does this PR introduce _any_ user-facing change?
No, this a new feature in Spark 4.0
### How was this patch tested?
Existing tests
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #48552 from gengliangwang/import_log_schema.
Lead-authored-by: Gengliang Wang <ge...@apache.org>
Co-authored-by: Hyukjin Kwon <gu...@apache.org>
Signed-off-by: Gengliang Wang <ge...@apache.org>
---
.../src/main/scala/org/apache/spark/util/LogUtils.scala | 4 ++--
docs/configuration.md | 15 ++++++++-------
python/pyspark/logger/__init__.py | 8 ++------
python/pyspark/logger/logger.py | 10 ++++++++++
python/pyspark/shell.py | 1 +
python/pyspark/util.py | 16 ----------------
.../main/scala/org/apache/spark/repl/SparkILoop.scala | 3 ++-
.../test/scala/org/apache/spark/sql/LogQuerySuite.scala | 8 ++++++--
8 files changed, 31 insertions(+), 34 deletions(-)
diff --git a/common/utils/src/main/scala/org/apache/spark/util/LogUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/LogUtils.scala
index 5a798ffad3a9..8b41f1033927 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/LogUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/LogUtils.scala
@@ -29,9 +29,9 @@ object LogUtils {
/**
* Schema for structured Spark logs.
* Example usage:
- * val logDf = spark.read.schema(LOG_SCHEMA).json("path/to/logs")
+ * val logDf = spark.read.schema(SPARK_LOG_SCHEMA).json("path/to/logs")
*/
- val LOG_SCHEMA: String = """
+ val SPARK_LOG_SCHEMA: String = """
|ts TIMESTAMP,
|level STRING,
|msg STRING,
diff --git a/docs/configuration.md b/docs/configuration.md
index 3c83ed92c128..e095ae7a61b2 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -3761,21 +3761,22 @@ Starting from version 4.0.0, `spark-submit` has adopted the [JSON Template Layou
To configure the layout of structured logging, start with the `log4j2.properties.template` file.
-To query Spark logs using Spark SQL, you can use the following Python code snippet:
+To query Spark logs using Spark SQL, you can use the following code snippets:
+**Python:**
```python
-from pyspark.util import LogUtils
+from pyspark.logger import SPARK_LOG_SCHEMA
-logDf = spark.read.schema(LogUtils.LOG_SCHEMA).json("path/to/logs")
+logDf = spark.read.schema(SPARK_LOG_SCHEMA).json("path/to/logs")
```
-Or using the following Scala code snippet:
+**Scala:**
```scala
-import org.apache.spark.util.LogUtils.LOG_SCHEMA
+import org.apache.spark.util.LogUtils.SPARK_LOG_SCHEMA
-val logDf = spark.read.schema(LOG_SCHEMA).json("path/to/logs")
+val logDf = spark.read.schema(SPARK_LOG_SCHEMA).json("path/to/logs")
```
-
+**Note**: If you're using the interactive shell (pyspark shell or spark-shell), you can omit the import statement in the code because SPARK_LOG_SCHEMA is already available in the shell's context.
## Plain Text Logging
If you prefer plain text logging, you have two options:
- Disable structured JSON logging by setting the Spark configuration `spark.log.structuredLogging.enabled` to `false`.
diff --git a/python/pyspark/logger/__init__.py b/python/pyspark/logger/__init__.py
index d8fab8beca8d..9e629971f0cb 100644
--- a/python/pyspark/logger/__init__.py
+++ b/python/pyspark/logger/__init__.py
@@ -18,10 +18,6 @@
"""
PySpark logging
"""
-from pyspark.logger.logger import ( # noqa: F401
- PySparkLogger,
-)
+from pyspark.logger.logger import PySparkLogger, SPARK_LOG_SCHEMA # noqa: F401
-__all__ = [
- "PySparkLogger",
-]
+__all__ = ["PySparkLogger", "SPARK_LOG_SCHEMA"]
diff --git a/python/pyspark/logger/logger.py b/python/pyspark/logger/logger.py
index 975441a9cb57..a2226fd717e0 100644
--- a/python/pyspark/logger/logger.py
+++ b/python/pyspark/logger/logger.py
@@ -20,6 +20,16 @@ import logging
import json
from typing import cast, Optional
+SPARK_LOG_SCHEMA = (
+ "ts TIMESTAMP, "
+ "level STRING, "
+ "msg STRING, "
+ "context map<STRING, STRING>, "
+ "exception STRUCT<class STRING, msg STRING, "
+ "stacktrace ARRAY<STRUCT<class STRING, method STRING, file STRING,line STRING>>>,"
+ "logger STRING"
+)
+
class JSONFormatter(logging.Formatter):
"""
diff --git a/python/pyspark/shell.py b/python/pyspark/shell.py
index 12ff86ecc9ff..91951b644f6b 100644
--- a/python/pyspark/shell.py
+++ b/python/pyspark/shell.py
@@ -30,6 +30,7 @@ import sys
import pyspark
from pyspark.core.context import SparkContext
+from pyspark.logger import SPARK_LOG_SCHEMA # noqa: F401
from pyspark.sql import SparkSession
from pyspark.sql.context import SQLContext
from pyspark.sql.utils import is_remote
diff --git a/python/pyspark/util.py b/python/pyspark/util.py
index cca44435efe6..10d68e9fd8bf 100644
--- a/python/pyspark/util.py
+++ b/python/pyspark/util.py
@@ -118,22 +118,6 @@ class VersionUtils:
)
-class LogUtils:
- """
- Utils for querying structured Spark logs with Spark SQL.
- """
-
- LOG_SCHEMA = (
- "ts TIMESTAMP, "
- "level STRING, "
- "msg STRING, "
- "context map<STRING, STRING>, "
- "exception STRUCT<class STRING, msg STRING, "
- "stacktrace ARRAY<STRUCT<class STRING, method STRING, file STRING,line STRING>>>,"
- "logger STRING"
- )
-
-
def fail_on_stopiteration(f: Callable) -> Callable:
"""
Wraps the input function to fail on 'StopIteration' by raising a 'RuntimeError'
diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
index 861cf5a740ce..f49e8adcc74a 100644
--- a/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
+++ b/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
@@ -66,7 +66,8 @@ class SparkILoop(in0: BufferedReader, out: PrintWriter)
"import org.apache.spark.SparkContext._",
"import spark.implicits._",
"import spark.sql",
- "import org.apache.spark.sql.functions._"
+ "import org.apache.spark.sql.functions._",
+ "import org.apache.spark.util.LogUtils.SPARK_LOG_SCHEMA"
)
override protected def internalReplAutorunCode(): Seq[String] =
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/LogQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/LogQuerySuite.scala
index df0fbf15a98e..873337e7a424 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/LogQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/LogQuerySuite.scala
@@ -21,7 +21,7 @@ import java.io.File
import org.apache.spark.internal.{Logging, LogKeys, MDC}
import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.util.LogUtils.LOG_SCHEMA
+import org.apache.spark.util.LogUtils.SPARK_LOG_SCHEMA
/**
* Test suite for querying Spark logs using SQL.
@@ -42,7 +42,11 @@ class LogQuerySuite extends QueryTest with SharedSparkSession with Logging {
}
private def createTempView(viewName: String): Unit = {
- spark.read.schema(LOG_SCHEMA).json(logFile.getCanonicalPath).createOrReplaceTempView(viewName)
+ spark
+ .read
+ .schema(SPARK_LOG_SCHEMA)
+ .json(logFile.getCanonicalPath)
+ .createOrReplaceTempView(viewName)
}
test("Query Spark logs using SQL") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org