You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ch...@apache.org on 2026/09/14 06:09:22 UTC
(spark) branch branch-4.1 updated: [SPARK-59081][SQL] HiveSessionStateBuilder does not apply injectHintResolutionRule
This is an automated email from the ASF dual-hosted git repository.
pan3793 pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new c94ffbc6ca7f [SPARK-59081][SQL] HiveSessionStateBuilder does not apply injectHintResolutionRule
c94ffbc6ca7f is described below
commit c94ffbc6ca7fb38ae0a521508980f33252ec1e20
Author: Simon Maling <s....@celonis.com>
AuthorDate: Mon Sep 14 14:08:08 2026 +0800
[SPARK-59081][SQL] HiveSessionStateBuilder does not apply injectHintResolutionRule
### What changes were proposed in this pull request?
https://issues.apache.org/jira/browse/SPARK-59081
`HiveSessionStateBuilder` constructs its own `Analyzer` and does not override `hintResolutionRules`. The Catalyst default is `Nil`, so `SparkSessionExtensions.injectHintResolutionRule` is ignored when Hive support is enabled (`enableHiveSupport()`, Hive Thrift Server / `jdbc:hive2`).
`BaseSessionStateBuilder` already wires:
```scala
override val hintResolutionRules: Seq[Rule[LogicalPlan]] =
customHintResolutionRules
```
This PR copies that override onto the Hive analyzer.
`injectResolutionRule` is not a substitute: Hive's `extendedResolutionRules` run `ResolveSQLOnFile` *before* `customResolutionRules`. This is the same class of miss as SPARK-56453 (Hive analyzer lists not updated when the in-memory builder was).
### Why are the changes needed?
With `enableHiveSupport()`, extensions that inject analyzer rules into the Hints batch never run. That includes any early-analysis work that must happen before `ResolveSQLOnFile` (for example path SQL such as `SELECT * FROM parquet.\`s3://...\``). In-memory sessions and Spark Connect without Hive are unaffected.
### Does this PR introduce _any_ user-facing change?
Yes. `SparkSession.builder.enableHiveSupport().withExtensions(_.injectHintResolutionRule(...))` now applies those rules, matching the non-Hive session builder. Previously they were silently dropped.
### How was this patch tested?
Added `HiveSparkSessionExtensionSuite` (isolated Hive `SparkSession`, no `TestHiveSingleton`):
- SPARK-59081: inject custom hint rule (`CONVERT_TO_EMPTY` -> `LocalRelation`), mirroring `SparkSessionExtensionSuite`.
- SPARK-59081: path SQL `SELECT * FROM parquet.\`...\``: a hint rule sees `UnresolvedRelation`; an `injectResolutionRule` does not (because Hive places `ResolveSQLOnFile` before `customResolutionRules`).
Locally: `build/sbt -Phive 'hive/testOnly org.apache.spark.sql.hive.HiveSparkSessionExtensionSuite'` (Java 17) — both tests passed.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Cursor Grok 4.6
Closes #58383 from s-maling-celonis/hive-hint-resolution-rules.
Authored-by: Simon Maling <s....@celonis.com>
Signed-off-by: Cheng Pan <ch...@apache.org>
(cherry picked from commit 0cefd07b9ac640cbfc184311bd7d01ef1f9dc9cf)
Signed-off-by: Cheng Pan <ch...@apache.org>
---
.../spark/sql/hive/HiveSessionStateBuilder.scala | 3 +
.../sql/hive/HiveSparkSessionExtensionSuite.scala | 138 +++++++++++++++++++++
2 files changed, 141 insertions(+)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
index dec947651dd6..2d05df0d613e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
@@ -86,6 +86,9 @@ class HiveSessionStateBuilder(
* A logical query plan `Analyzer` with rules specific to Hive.
*/
override protected def analyzer: Analyzer = new Analyzer(catalogManager, sharedRelationCache) {
+ override val hintResolutionRules: Seq[Rule[LogicalPlan]] =
+ customHintResolutionRules
+
override val singlePassResolverExtensions: Seq[ResolverExtension] = Seq(
new LogicalRelationResolver,
new HiveTableRelationNoopResolver
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSessionExtensionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSessionExtensionSuite.scala
new file mode 100644
index 000000000000..98bb6563c9c8
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSessionExtensionSuite.scala
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive
+
+import java.util.Locale
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.internal.config.UI
+import org.apache.spark.sql.{SparkSession => SqlSession, SparkSessionExtensions}
+import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, UnresolvedHint}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.classic.SparkSession
+import org.apache.spark.util.Utils
+
+/**
+ * HiveSessionStateBuilder replaces Analyzer and must copy extension hook lists
+ * from BaseSessionStateBuilder. Without hintResolutionRules, injectHintResolutionRule
+ * is a no-op under enableHiveSupport() (Hive Thrift / jdbc:hive2).
+ *
+ * SPARK-59081: Do not use TestHiveSingleton: that session is already built, so
+ * withExtensions on a new builder is required.
+ */
+class HiveSparkSessionExtensionSuite extends SparkFunSuite {
+
+ // Isolated sessions leave a SparkContext running; do not treat that as a leak.
+ override protected val enableAutoThreadAudit = false
+
+ private def withHiveSession(
+ builders: Seq[SparkSessionExtensions => Unit])(f: SparkSession => Unit): Unit = {
+ val savedActive = SparkSession.getActiveSession
+ val savedDefault = SparkSession.getDefaultSession
+ val builder = SparkSession.builder()
+ .master("local[1]")
+ .config(UI.UI_ENABLED.key, false)
+ .enableHiveSupport()
+ HiveUtils.newTemporaryConfiguration(useInMemoryDerby = true).foreach {
+ case (k, v) => builder.config(k, v)
+ }
+ builders.foreach(builder.withExtensions)
+ val session = try {
+ SparkSession.clearActiveSession()
+ SparkSession.clearDefaultSession()
+ builder.create()
+ } finally {
+ savedDefault.foreach(SparkSession.setDefaultSession)
+ savedActive.foreach(SparkSession.setActiveSession)
+ }
+ try {
+ f(session)
+ } finally {
+ SparkSession.clearActiveSession()
+ SparkSession.clearDefaultSession()
+ savedDefault.foreach(SparkSession.setDefaultSession)
+ savedActive.foreach(SparkSession.setActiveSession)
+ }
+ }
+
+ case class MyHintRule(spark: SqlSession) extends Rule[LogicalPlan] {
+ val myHintName = Set("CONVERT_TO_EMPTY")
+
+ override def apply(plan: LogicalPlan): LogicalPlan =
+ plan.resolveOperators {
+ case h: UnresolvedHint if myHintName.contains(h.name.toUpperCase(Locale.ROOT)) =>
+ LocalRelation(h.output, data = Seq.empty, isStreaming = h.isStreaming)
+ }
+ }
+
+ test("SPARK-59081: inject custom hint rule") {
+ withHiveSession(Seq(_.injectHintResolutionRule(MyHintRule))) { session =>
+ assert(session.sessionState.catalog.getClass.getName.contains("HiveSessionCatalog"))
+ assert(session.sessionState.analyzer.hintResolutionRules.contains(MyHintRule(session)))
+ assert(
+ session.range(1).hint("CONVERT_TO_EMPTY").logicalPlan.isInstanceOf[LocalRelation],
+ "plan is expected to be a local relation")
+ }
+ }
+
+ test("SPARK-59081: hint rule sees UnresolvedRelation for path SQL before ResolveSQLOnFile") {
+ var hintSawUnresolved = false
+ var resolutionSawUnresolved = false
+
+ case class PathSqlHintRule(spark: SqlSession) extends Rule[LogicalPlan] {
+ override def apply(plan: LogicalPlan): LogicalPlan = {
+ plan.foreach {
+ case r: UnresolvedRelation
+ if r.multipartIdentifier.size == 2 &&
+ r.multipartIdentifier.head.equalsIgnoreCase("parquet") =>
+ hintSawUnresolved = true
+ case _ =>
+ }
+ plan
+ }
+ }
+
+ case class PathSqlResolutionRule(spark: SqlSession) extends Rule[LogicalPlan] {
+ override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperators {
+ case r: UnresolvedRelation
+ if r.multipartIdentifier.size == 2 &&
+ r.multipartIdentifier.head.equalsIgnoreCase("parquet") =>
+ resolutionSawUnresolved = true
+ r
+ }
+ }
+
+ withHiveSession(Seq(
+ _.injectHintResolutionRule(PathSqlHintRule),
+ _.injectResolutionRule(PathSqlResolutionRule))) { session =>
+ val dir = Utils.createTempDir()
+ try {
+ val path = new java.io.File(dir, "data").getCanonicalPath
+ session.range(1).write.parquet(path)
+ val df = session.sql(s"SELECT * FROM parquet.`$path`")
+ df.queryExecution.analyzed
+ assert(hintSawUnresolved,
+ "injectHintResolutionRule must run before ResolveSQLOnFile")
+ assert(!resolutionSawUnresolved,
+ "injectResolutionRule is after ResolveSQLOnFile and must not see the path relation")
+ } finally {
+ Utils.deleteRecursively(dir)
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org