You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ge...@apache.org on 2023/03/29 08:16:16 UTC
[spark] branch master updated: [SPARK-42952][SQL] Simplify the parameter of analyzer rule PreprocessTableCreation and DataSourceAnalysis
This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new d6a6af51d64 [SPARK-42952][SQL] Simplify the parameter of analyzer rule PreprocessTableCreation and DataSourceAnalysis
d6a6af51d64 is described below
commit d6a6af51d64e809c7413fde05fdebdd035d89f2e
Author: Gengliang Wang <ge...@apache.org>
AuthorDate: Wed Mar 29 01:15:57 2023 -0700
[SPARK-42952][SQL] Simplify the parameter of analyzer rule PreprocessTableCreation and DataSourceAnalysis
### What changes were proposed in this pull request?
Simplify the parameter of the following analyzer rule:
* PreprocessTableCreation: use a SessionCatalog instead of passing SparkSession
* DataSourceAnalysis: remove the unused `Analyzer` in the parameter and turn it from class to object.
### Why are the changes needed?
Code cleanup
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing tests
Closes #40580 from gengliangwang/catalog.
Authored-by: Gengliang Wang <ge...@apache.org>
Signed-off-by: Gengliang Wang <ge...@apache.org>
---
.../apache/spark/sql/execution/datasources/DataSourceStrategy.scala | 2 +-
.../main/scala/org/apache/spark/sql/execution/datasources/rules.scala | 4 +---
.../scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala | 4 ++--
.../apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala | 2 +-
.../scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala | 3 +--
.../scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala | 4 ++--
6 files changed, 8 insertions(+), 11 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
index e3a1f6f6b68..69c7624605b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
@@ -61,7 +61,7 @@ import org.apache.spark.unsafe.types.UTF8String
* Note that, this rule must be run after `PreprocessTableCreation` and
* `PreprocessTableInsertion`.
*/
-case class DataSourceAnalysis(analyzer: Analyzer) extends Rule[LogicalPlan] {
+object DataSourceAnalysis extends Rule[LogicalPlan] {
def resolver: Resolver = conf.resolver
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
index 635562ab54d..2564d7e50a2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
@@ -89,9 +89,7 @@ class ResolveSQLOnFile(sparkSession: SparkSession) extends Rule[LogicalPlan] {
/**
* Preprocess [[CreateTable]], to do some normalization and checking.
*/
-case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[LogicalPlan] {
- // catalog is a def and not a val/lazy val as the latter would introduce a circular reference
- private def catalog = sparkSession.sessionState.catalog
+case class PreprocessTableCreation(catalog: SessionCatalog) extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
// When we CREATE TABLE without specifying the table schema, we should fail the query if
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
index f17d0c3dd2e..8585808d762 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
@@ -193,9 +193,9 @@ abstract class BaseSessionStateBuilder(
override val postHocResolutionRules: Seq[Rule[LogicalPlan]] =
DetectAmbiguousSelfJoin +:
- PreprocessTableCreation(session) +:
+ PreprocessTableCreation(catalog) +:
PreprocessTableInsertion +:
- DataSourceAnalysis(this) +:
+ DataSourceAnalysis +:
ApplyCharTypePadding +:
ReplaceCharWithVarchar +:
customPostHocResolutionRules
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala
index 44cd4f0f9b3..a51ac78fdb2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala
@@ -44,7 +44,7 @@ class V2CommandsCaseSensitivitySuite
schema.toAttributes)
override protected def extendedAnalysisRules: Seq[Rule[LogicalPlan]] = {
- Seq(PreprocessTableCreation(spark))
+ Seq(PreprocessTableCreation(spark.sessionState.catalog))
}
test("CreateTableAsSelect: using top level field for partitioning") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala
index ba0dfa95178..4da7dbb93e4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.sources
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.analysis.SimpleAnalyzer
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, Cast, Expression, Literal}
import org.apache.spark.sql.catalyst.plans.SQLHelper
@@ -69,7 +68,7 @@ class DataSourceAnalysisSuite extends SparkFunSuite with SQLHelper {
Cast(e, dt, Option(SQLConf.get.sessionLocalTimeZone))
}
}
- val rule = DataSourceAnalysis(SimpleAnalyzer)
+ val rule = DataSourceAnalysis
testRule(
"convertStaticPartitions only handle INSERT having at least static partitions",
caseSensitive) {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
index c4e0057ae95..a3ee3e0fc33 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
@@ -98,9 +98,9 @@ class HiveSessionStateBuilder(
DetectAmbiguousSelfJoin +:
new DetermineTableStats(session) +:
RelationConversions(catalog) +:
- PreprocessTableCreation(session) +:
+ PreprocessTableCreation(catalog) +:
PreprocessTableInsertion +:
- DataSourceAnalysis(this) +:
+ DataSourceAnalysis +:
ApplyCharTypePadding +:
HiveAnalysis +:
ReplaceCharWithVarchar +:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org