You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ge...@apache.org on 2024/10/21 17:42:25 UTC

(spark) branch master updated: [SPARK-50036][CORE][PYTHON] Include SPARK_LOG_SCHEMA in the context of REPL shell

This is an automated email from the ASF dual-hosted git repository.

gengliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new f86df1e37b47 [SPARK-50036][CORE][PYTHON] Include SPARK_LOG_SCHEMA in the context of REPL shell
f86df1e37b47 is described below

commit f86df1e37b4770bffddafc0216ae5eb4b3ac25b7
Author: Gengliang Wang <ge...@apache.org>
AuthorDate: Mon Oct 21 10:42:13 2024 -0700

    [SPARK-50036][CORE][PYTHON] Include SPARK_LOG_SCHEMA in the context of REPL shell
    
    ### What changes were proposed in this pull request?
    
    Before the Change:
    
    Users needed to import LOG_SCHEMA to read structured logs as a JSON data source:
    ```
    import org.apache.spark.util.LogUtils.LOG_SCHEMA
    
    val logDf = spark.read.schema(LOG_SCHEMA).json("path/to/logs")
     ```
    
    After the Change:
    
    - Renamed for Clarity:`LOG_SCHEMA` has been renamed to `SPARK_LOG_SCHEMA` to make its purpose more clear.
    - No Import Needed in REPL Shells: You can now use `SPARK_LOG_SCHEMA` directly in REPL environments like spark-shell and pyspark without importing it.
    
    Now, you can read structured logs without the import:
    
    ```
    val logDf = spark.read.schema(SPARK_LOG_SCHEMA).json("path/to/logs")
    ```
    
    ### Why are the changes needed?
    
    Simply the way to query structured logs
    
    ### Does this PR introduce _any_ user-facing change?
    
    No, this a new feature in Spark 4.0
    ### How was this patch tested?
    
    Existing tests
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No
    
    Closes #48552 from gengliangwang/import_log_schema.
    
    Lead-authored-by: Gengliang Wang <ge...@apache.org>
    Co-authored-by: Hyukjin Kwon <gu...@apache.org>
    Signed-off-by: Gengliang Wang <ge...@apache.org>
---
 .../src/main/scala/org/apache/spark/util/LogUtils.scala  |  4 ++--
 docs/configuration.md                                    | 15 ++++++++-------
 python/pyspark/logger/__init__.py                        |  8 ++------
 python/pyspark/logger/logger.py                          | 10 ++++++++++
 python/pyspark/shell.py                                  |  1 +
 python/pyspark/util.py                                   | 16 ----------------
 .../main/scala/org/apache/spark/repl/SparkILoop.scala    |  3 ++-
 .../test/scala/org/apache/spark/sql/LogQuerySuite.scala  |  8 ++++++--
 8 files changed, 31 insertions(+), 34 deletions(-)

diff --git a/common/utils/src/main/scala/org/apache/spark/util/LogUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/LogUtils.scala
index 5a798ffad3a9..8b41f1033927 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/LogUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/LogUtils.scala
@@ -29,9 +29,9 @@ object LogUtils {
   /**
    * Schema for structured Spark logs.
    * Example usage:
-   *   val logDf = spark.read.schema(LOG_SCHEMA).json("path/to/logs")
+   *   val logDf = spark.read.schema(SPARK_LOG_SCHEMA).json("path/to/logs")
    */
-  val LOG_SCHEMA: String = """
+  val SPARK_LOG_SCHEMA: String = """
     |ts TIMESTAMP,
     |level STRING,
     |msg STRING,
diff --git a/docs/configuration.md b/docs/configuration.md
index 3c83ed92c128..e095ae7a61b2 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -3761,21 +3761,22 @@ Starting from version 4.0.0, `spark-submit` has adopted the [JSON Template Layou
 
 To configure the layout of structured logging, start with the `log4j2.properties.template` file.
 
-To query Spark logs using Spark SQL, you can use the following Python code snippet:
+To query Spark logs using Spark SQL, you can use the following code snippets:
 
+**Python:**
 ```python
-from pyspark.util import LogUtils
+from pyspark.logger import SPARK_LOG_SCHEMA
 
-logDf = spark.read.schema(LogUtils.LOG_SCHEMA).json("path/to/logs")
+logDf = spark.read.schema(SPARK_LOG_SCHEMA).json("path/to/logs")
 ```
 
-Or using the following Scala code snippet:
+**Scala:**
 ```scala
-import org.apache.spark.util.LogUtils.LOG_SCHEMA
+import org.apache.spark.util.LogUtils.SPARK_LOG_SCHEMA
 
-val logDf = spark.read.schema(LOG_SCHEMA).json("path/to/logs")
+val logDf = spark.read.schema(SPARK_LOG_SCHEMA).json("path/to/logs")
 ```
-
+**Note**: If you're using the interactive shell (pyspark shell or spark-shell), you can omit the import statement in the code because SPARK_LOG_SCHEMA is already available in the shell's context.
 ## Plain Text Logging
 If you prefer plain text logging, you have two options:
 - Disable structured JSON logging by setting the Spark configuration `spark.log.structuredLogging.enabled` to `false`.
diff --git a/python/pyspark/logger/__init__.py b/python/pyspark/logger/__init__.py
index d8fab8beca8d..9e629971f0cb 100644
--- a/python/pyspark/logger/__init__.py
+++ b/python/pyspark/logger/__init__.py
@@ -18,10 +18,6 @@
 """
 PySpark logging
 """
-from pyspark.logger.logger import (  # noqa: F401
-    PySparkLogger,
-)
+from pyspark.logger.logger import PySparkLogger, SPARK_LOG_SCHEMA  # noqa: F401
 
-__all__ = [
-    "PySparkLogger",
-]
+__all__ = ["PySparkLogger", "SPARK_LOG_SCHEMA"]
diff --git a/python/pyspark/logger/logger.py b/python/pyspark/logger/logger.py
index 975441a9cb57..a2226fd717e0 100644
--- a/python/pyspark/logger/logger.py
+++ b/python/pyspark/logger/logger.py
@@ -20,6 +20,16 @@ import logging
 import json
 from typing import cast, Optional
 
+SPARK_LOG_SCHEMA = (
+    "ts TIMESTAMP, "
+    "level STRING, "
+    "msg STRING, "
+    "context map<STRING, STRING>, "
+    "exception STRUCT<class STRING, msg STRING, "
+    "stacktrace ARRAY<STRUCT<class STRING, method STRING, file STRING,line STRING>>>,"
+    "logger STRING"
+)
+
 
 class JSONFormatter(logging.Formatter):
     """
diff --git a/python/pyspark/shell.py b/python/pyspark/shell.py
index 12ff86ecc9ff..91951b644f6b 100644
--- a/python/pyspark/shell.py
+++ b/python/pyspark/shell.py
@@ -30,6 +30,7 @@ import sys
 
 import pyspark
 from pyspark.core.context import SparkContext
+from pyspark.logger import SPARK_LOG_SCHEMA  # noqa: F401
 from pyspark.sql import SparkSession
 from pyspark.sql.context import SQLContext
 from pyspark.sql.utils import is_remote
diff --git a/python/pyspark/util.py b/python/pyspark/util.py
index cca44435efe6..10d68e9fd8bf 100644
--- a/python/pyspark/util.py
+++ b/python/pyspark/util.py
@@ -118,22 +118,6 @@ class VersionUtils:
             )
 
 
-class LogUtils:
-    """
-    Utils for querying structured Spark logs with Spark SQL.
-    """
-
-    LOG_SCHEMA = (
-        "ts TIMESTAMP, "
-        "level STRING, "
-        "msg STRING, "
-        "context map<STRING, STRING>, "
-        "exception STRUCT<class STRING, msg STRING, "
-        "stacktrace ARRAY<STRUCT<class STRING, method STRING, file STRING,line STRING>>>,"
-        "logger STRING"
-    )
-
-
 def fail_on_stopiteration(f: Callable) -> Callable:
     """
     Wraps the input function to fail on 'StopIteration' by raising a 'RuntimeError'
diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
index 861cf5a740ce..f49e8adcc74a 100644
--- a/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
+++ b/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
@@ -66,7 +66,8 @@ class SparkILoop(in0: BufferedReader, out: PrintWriter)
     "import org.apache.spark.SparkContext._",
     "import spark.implicits._",
     "import spark.sql",
-    "import org.apache.spark.sql.functions._"
+    "import org.apache.spark.sql.functions._",
+    "import org.apache.spark.util.LogUtils.SPARK_LOG_SCHEMA"
   )
 
   override protected def internalReplAutorunCode(): Seq[String] =
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/LogQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/LogQuerySuite.scala
index df0fbf15a98e..873337e7a424 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/LogQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/LogQuerySuite.scala
@@ -21,7 +21,7 @@ import java.io.File
 
 import org.apache.spark.internal.{Logging, LogKeys, MDC}
 import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.util.LogUtils.LOG_SCHEMA
+import org.apache.spark.util.LogUtils.SPARK_LOG_SCHEMA
 
 /**
  * Test suite for querying Spark logs using SQL.
@@ -42,7 +42,11 @@ class LogQuerySuite extends QueryTest with SharedSparkSession with Logging {
   }
 
   private def createTempView(viewName: String): Unit = {
-    spark.read.schema(LOG_SCHEMA).json(logFile.getCanonicalPath).createOrReplaceTempView(viewName)
+    spark
+      .read
+      .schema(SPARK_LOG_SCHEMA)
+      .json(logFile.getCanonicalPath)
+      .createOrReplaceTempView(viewName)
   }
 
   test("Query Spark logs using SQL") {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org