You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2024/09/19 19:19:23 UTC
(spark) branch branch-3.4 updated: [SPARK-46535][SQL][3.4] Fix NPE when describe extended a column without col stats
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new cb89d18a4d75 [SPARK-46535][SQL][3.4] Fix NPE when describe extended a column without col stats
cb89d18a4d75 is described below
commit cb89d18a4d750fc88e5d747601352488223e97b5
Author: saitharun15 <sa...@gmail.com>
AuthorDate: Thu Sep 19 12:19:10 2024 -0700
[SPARK-46535][SQL][3.4] Fix NPE when describe extended a column without col stats
### What changes were proposed in this pull request?
Backport [#44524 ] to 3.4 for [[SPARK-46535]](https://issues.apache.org/jira/browse/SPARK-46535)[SQL] Fix NPE when describe extended a column without col stats
### Why are the changes needed?
Currently executing DESCRIBE TABLE EXTENDED a column without col stats with v2 table will throw a null pointer exception.
```
Cannot invoke "org.apache.spark.sql.connector.read.colstats.ColumnStatistics.min()" because the return value of "scala.Option.get()" is null
java.lang.NullPointerException: Cannot invoke "org.apache.spark.sql.connector.read.colstats.ColumnStatistics.min()" because the return value of "scala.Option.get()" is null
at org.apache.spark.sql.execution.datasources.v2.DescribeColumnExec.run(DescribeColumnExec.scala:63)
at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result$lzycompute(V2CommandExec.scala:43)
at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result(V2CommandExec.scala:43)
at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.executeCollect(V2CommandExec.scala:49)
at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:98)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$6(SQLExecution.scala:118)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:195)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:103)
```
### Does this PR introduce _any_ user-facing change?
### How was this patch tested?
Add a new test describe extended (formatted) a column without col stats
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #48160 from saitharun15/SPARK-46535-branch-3.4.
Lead-authored-by: saitharun15 <sa...@gmail.com>
Co-authored-by: Sai Tharun <ps...@ibm.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../datasources/v2/DescribeColumnExec.scala | 2 +-
.../execution/command/v2/DescribeTableSuite.scala | 21 +++++++++++++++++++++
2 files changed, 22 insertions(+), 1 deletion(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeColumnExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeColumnExec.scala
index 61ccda3fc954..2683d8d547f0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeColumnExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeColumnExec.scala
@@ -53,7 +53,7 @@ case class DescribeColumnExec(
read.newScanBuilder(CaseInsensitiveStringMap.empty()).build() match {
case s: SupportsReportStatistics =>
val stats = s.estimateStatistics()
- Some(stats.columnStats().get(FieldReference.column(column.name)))
+ Option(stats.columnStats().get(FieldReference.column(column.name)))
case _ => None
}
case _ => None
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala
index 25363dcea699..a12bb92072bc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala
@@ -175,4 +175,25 @@ class DescribeTableSuite extends command.DescribeTableSuiteBase
Row("max_col_len", "NULL")))
}
}
+
+ test("SPARK-46535: describe extended (formatted) a column without col stats") {
+ withNamespaceAndTable("ns", "tbl") { tbl =>
+ sql(
+ s"""
+ |CREATE TABLE $tbl
+ |(key INT COMMENT 'column_comment', col STRING)
+ |$defaultUsing""".stripMargin)
+
+ val descriptionDf = sql(s"DESCRIBE TABLE EXTENDED $tbl key")
+ assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === Seq(
+ ("info_name", StringType),
+ ("info_value", StringType)))
+ QueryTest.checkAnswer(
+ descriptionDf,
+ Seq(
+ Row("col_name", "key"),
+ Row("data_type", "int"),
+ Row("comment", "column_comment")))
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org