You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2020/03/05 10:43:00 UTC
[spark] branch branch-3.0 updated: [SPARK-31024][SQL] Allow
specifying session catalog name `spark_catalog` in qualified column names
for v1 tables
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 4fcb5ae [SPARK-31024][SQL] Allow specifying session catalog name `spark_catalog` in qualified column names for v1 tables
4fcb5ae is described below
commit 4fcb5ae22623de96247fcfe7341be0af7ed2471f
Author: Terry Kim <yu...@gmail.com>
AuthorDate: Thu Mar 5 18:33:59 2020 +0800
[SPARK-31024][SQL] Allow specifying session catalog name `spark_catalog` in qualified column names for v1 tables
### What changes were proposed in this pull request?
Currently, the user cannot specify the session catalog name (`spark_catalog`) in qualified column names for v1 tables:
```
SELECT spark_catalog.default.t.i FROM spark_catalog.default.t
```
fails with `cannot resolve 'spark_catalog.default.t.i`.
This is inconsistent with v2 table behavior where catalog name can be used:
```
SELECT testcat.ns1.tbl.id FROM testcat.ns1.tbl.id
```
This PR proposes to fix the inconsistency and allow the user to specify session catalog name in column names for v1 tables.
### Why are the changes needed?
Fixing an inconsistent behavior.
### Does this PR introduce any user-facing change?
Yes, now the following query works:
```
SELECT spark_catalog.default.t.i FROM spark_catalog.default.t
```
### How was this patch tested?
Added new tests.
Closes #27776 from imback82/spark_catalog_col_name_resolution.
Authored-by: Terry Kim <yu...@gmail.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit 66b4fd040e97cb6de6536a5545017278879c98fb)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../spark/sql/catalyst/analysis/Analyzer.scala | 2 +-
.../sql/catalyst/catalog/SessionCatalog.scala | 6 +-
.../spark/sql/catalyst/expressions/package.scala | 102 ++++++++++++++++-----
.../sql/catalyst/catalog/SessionCatalogSuite.scala | 6 +-
.../results/columnresolution-negative.sql.out | 26 +++---
.../results/postgreSQL/create_view.sql.out | 2 +-
.../sql-tests/results/postgreSQL/join.sql.out | 2 +-
.../results/postgreSQL/select_having.sql.out | 2 +-
.../results/postgreSQL/window_part3.sql.out | 4 +-
.../results/udf/postgreSQL/udf-join.sql.out | 2 +-
.../udf/postgreSQL/udf-select_having.sql.out | 2 +-
.../scala/org/apache/spark/sql/SQLQuerySuite.scala | 4 +-
.../spark/sql/connector/DataSourceV2SQLSuite.scala | 8 +-
.../spark/sql/hive/HiveMetastoreCatalogSuite.scala | 2 +-
14 files changed, 111 insertions(+), 59 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 254dd44..3cb754d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -935,7 +935,7 @@ class Analyzer(
v1SessionCatalog.getRelation(v1Table.v1Table)
case table =>
SubqueryAlias(
- ident.asMultipartIdentifier,
+ catalog.name +: ident.asMultipartIdentifier,
DataSourceV2Relation.create(table, Some(catalog), Some(ident)))
}
val key = catalog.name +: ident.namespace :+ ident.name
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index c80d9d2..3a63aff 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo, Im
import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParserInterface}
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias, View}
import org.apache.spark.sql.catalyst.util.StringUtils
+import org.apache.spark.sql.connector.catalog.CatalogManager
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.StaticSQLConf.GLOBAL_TEMP_DATABASE
import org.apache.spark.sql.types.StructType
@@ -758,6 +759,7 @@ class SessionCatalog(
val name = metadata.identifier
val db = formatDatabaseName(name.database.getOrElse(currentDb))
val table = formatTableName(name.table)
+ val multiParts = Seq(CatalogManager.SESSION_CATALOG_NAME, db, table)
if (metadata.tableType == CatalogTableType.VIEW) {
val viewText = metadata.viewText.getOrElse(sys.error("Invalid view without text."))
@@ -769,9 +771,9 @@ class SessionCatalog(
desc = metadata,
output = metadata.schema.toAttributes,
child = parser.parsePlan(viewText))
- SubqueryAlias(table, db, child)
+ SubqueryAlias(multiParts, child)
} else {
- SubqueryAlias(table, db, UnresolvedCatalogRelation(metadata))
+ SubqueryAlias(multiParts, UnresolvedCatalogRelation(metadata))
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
index 9f42e64..1b59056 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
@@ -142,22 +142,44 @@ package object expressions {
}
/** Map to use for qualified case insensitive attribute lookups with 3 part key */
- @transient private val qualified3Part: Map[(String, String, String), Seq[Attribute]] = {
+ @transient private lazy val qualified3Part: Map[(String, String, String), Seq[Attribute]] = {
// key is 3 part: database name, table name and name
- val grouped = attrs.filter(_.qualifier.length == 2).groupBy { a =>
- (a.qualifier.head.toLowerCase(Locale.ROOT),
- a.qualifier.last.toLowerCase(Locale.ROOT),
- a.name.toLowerCase(Locale.ROOT))
+ val grouped = attrs.filter(a => a.qualifier.length >= 2 && a.qualifier.length <= 3)
+ .groupBy { a =>
+ val qualifier = if (a.qualifier.length == 2) {
+ a.qualifier
+ } else {
+ a.qualifier.takeRight(2)
+ }
+ (qualifier.head.toLowerCase(Locale.ROOT),
+ qualifier.last.toLowerCase(Locale.ROOT),
+ a.name.toLowerCase(Locale.ROOT))
+ }
+ unique(grouped)
+ }
+
+ /** Map to use for qualified case insensitive attribute lookups with 4 part key */
+ @transient
+ private lazy val qualified4Part: Map[(String, String, String, String), Seq[Attribute]] = {
+ // key is 4 part: catalog name, database name, table name and name
+ val grouped = attrs.filter(_.qualifier.length == 3).groupBy { a =>
+ a.qualifier match {
+ case Seq(catalog, db, tbl) =>
+ (catalog.toLowerCase(Locale.ROOT),
+ db.toLowerCase(Locale.ROOT),
+ tbl.toLowerCase(Locale.ROOT),
+ a.name.toLowerCase(Locale.ROOT))
+ }
}
unique(grouped)
}
- /** Returns true if all qualifiers in `attrs` have 2 or less parts. */
- @transient private val hasTwoOrLessQualifierParts: Boolean =
- attrs.forall(_.qualifier.length <= 2)
+ /** Returns true if all qualifiers in `attrs` have 3 or less parts. */
+ @transient private val hasThreeOrLessQualifierParts: Boolean =
+ attrs.forall(_.qualifier.length <= 3)
- /** Match attributes for the case where all qualifiers in `attrs` have 2 or less parts. */
- private def matchWithTwoOrLessQualifierParts(
+ /** Match attributes for the case where all qualifiers in `attrs` have 3 or less parts. */
+ private def matchWithThreeOrLessQualifierParts(
nameParts: Seq[String],
resolver: Resolver): (Seq[Attribute], Seq[String]) = {
// Collect matching attributes given a name and a lookup.
@@ -167,25 +189,55 @@ package object expressions {
}
}
- // Find matches for the given name assuming that the 1st two parts are qualifier
- // (i.e. database name and table name) and the 3rd part is the actual column name.
+ // Find matches for the given name assuming that the 1st three parts are qualifier
+ // (i.e. catalog name, database name and table name) and the 4th part is the actual
+ // column name.
//
- // For example, consider an example where "db1" is the database name, "a" is the table name
- // and "b" is the column name and "c" is the struct field name.
- // If the name parts is db1.a.b.c, then Attribute will match
- // Attribute(b, qualifier("db1,"a")) and List("c") will be the second element
+ // For example, consider an example where "cat" is the catalog name, "db1" is the database
+ // name, "a" is the table name and "b" is the column name and "c" is the struct field name.
+ // If the name parts is cat.db1.a.b.c, then Attribute will match
+ // Attribute(b, qualifier("cat", "db1, "a")) and List("c") will be the second element
var matches: (Seq[Attribute], Seq[String]) = nameParts match {
- case dbPart +: tblPart +: name +: nestedFields =>
- val key = (dbPart.toLowerCase(Locale.ROOT),
+ case catalogPart +: dbPart +: tblPart +: name +: nestedFields =>
+ val key = (catalogPart.toLowerCase(Locale.ROOT), dbPart.toLowerCase(Locale.ROOT),
tblPart.toLowerCase(Locale.ROOT), name.toLowerCase(Locale.ROOT))
- val attributes = collectMatches(name, qualified3Part.get(key)).filter {
- a => (resolver(dbPart, a.qualifier.head) && resolver(tblPart, a.qualifier.last))
+ val attributes = collectMatches(name, qualified4Part.get(key)).filter { a =>
+ assert(a.qualifier.length == 3)
+ resolver(catalogPart, a.qualifier(0)) && resolver(dbPart, a.qualifier(1)) &&
+ resolver(tblPart, a.qualifier(2))
}
(attributes, nestedFields)
case _ =>
(Seq.empty, Seq.empty)
}
+ // Find matches for the given name assuming that the 1st two parts are qualifier
+ // (i.e. database name and table name) and the 3rd part is the actual column name.
+ //
+ // For example, consider an example where "db1" is the database name, "a" is the table name
+ // and "b" is the column name and "c" is the struct field name.
+ // If the name parts is db1.a.b.c, then it can match both
+ // Attribute(b, qualifier("cat", "db1, "a")) and Attribute(b, qualifier("db1, "a")),
+ // and List("c") will be the second element
+ if (matches._1.isEmpty) {
+ matches = nameParts match {
+ case dbPart +: tblPart +: name +: nestedFields =>
+ val key = (dbPart.toLowerCase(Locale.ROOT),
+ tblPart.toLowerCase(Locale.ROOT), name.toLowerCase(Locale.ROOT))
+ val attributes = collectMatches(name, qualified3Part.get(key)).filter { a =>
+ val qualifier = if (a.qualifier.length == 2) {
+ a.qualifier
+ } else {
+ a.qualifier.takeRight(2)
+ }
+ resolver(dbPart, qualifier.head) && resolver(tblPart, qualifier.last)
+ }
+ (attributes, nestedFields)
+ case _ =>
+ (Seq.empty, Seq.empty)
+ }
+ }
+
// If there are no matches, then find matches for the given name assuming that
// the 1st part is a qualifier (i.e. table name, alias, or subquery alias) and the
// 2nd part is the actual name. This returns a tuple of
@@ -219,9 +271,9 @@ package object expressions {
}
/**
- * Match attributes for the case where at least one qualifier in `attrs` has more than 2 parts.
+ * Match attributes for the case where at least one qualifier in `attrs` has more than 3 parts.
*/
- private def matchWithThreeOrMoreQualifierParts(
+ private def matchWithFourOrMoreQualifierParts(
nameParts: Seq[String],
resolver: Resolver): (Seq[Attribute], Seq[String]) = {
// Returns true if the `short` qualifier is a subset of the last elements of
@@ -277,10 +329,10 @@ package object expressions {
/** Perform attribute resolution given a name and a resolver. */
def resolve(nameParts: Seq[String], resolver: Resolver): Option[NamedExpression] = {
- val (candidates, nestedFields) = if (hasTwoOrLessQualifierParts) {
- matchWithTwoOrLessQualifierParts(nameParts, resolver)
+ val (candidates, nestedFields) = if (hasThreeOrLessQualifierParts) {
+ matchWithThreeOrLessQualifierParts(nameParts, resolver)
} else {
- matchWithThreeOrMoreQualifierParts(nameParts, resolver)
+ matchWithFourOrMoreQualifierParts(nameParts, resolver)
}
def name = UnresolvedAttribute(nameParts).name
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
index 0d9e2f6..4d88a8d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
@@ -635,11 +635,11 @@ abstract class SessionCatalogSuite extends AnalysisTest {
val view = View(desc = metadata, output = metadata.schema.toAttributes,
child = CatalystSqlParser.parsePlan(metadata.viewText.get))
comparePlans(catalog.lookupRelation(TableIdentifier("view1", Some("db3"))),
- SubqueryAlias("view1", "db3", view))
+ SubqueryAlias(Seq(CatalogManager.SESSION_CATALOG_NAME, "db3", "view1"), view))
// Look up a view using current database of the session catalog.
catalog.setCurrentDatabase("db3")
comparePlans(catalog.lookupRelation(TableIdentifier("view1")),
- SubqueryAlias("view1", "db3", view))
+ SubqueryAlias(Seq(CatalogManager.SESSION_CATALOG_NAME, "db3", "view1"), view))
}
}
@@ -655,7 +655,7 @@ abstract class SessionCatalogSuite extends AnalysisTest {
val view = View(desc = metadata, output = metadata.schema.toAttributes,
child = CatalystSqlParser.parsePlan(metadata.viewText.get))
comparePlans(catalog.lookupRelation(TableIdentifier("view2", Some("db3"))),
- SubqueryAlias("view2", "db3", view))
+ SubqueryAlias(Seq(CatalogManager.SESSION_CATALOG_NAME, "db3", "view2"), view))
}
}
diff --git a/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out b/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out
index f34b75a..04ddfe0 100644
--- a/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out
@@ -72,7 +72,7 @@ SELECT i1 FROM t1, mydb1.t1
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-Reference 'i1' is ambiguous, could be: mydb1.t1.i1, mydb1.t1.i1.; line 1 pos 7
+Reference 'i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7
-- !query
@@ -81,7 +81,7 @@ SELECT t1.i1 FROM t1, mydb1.t1
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-Reference 't1.i1' is ambiguous, could be: mydb1.t1.i1, mydb1.t1.i1.; line 1 pos 7
+Reference 't1.i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7
-- !query
@@ -90,7 +90,7 @@ SELECT mydb1.t1.i1 FROM t1, mydb1.t1
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-Reference 'mydb1.t1.i1' is ambiguous, could be: mydb1.t1.i1, mydb1.t1.i1.; line 1 pos 7
+Reference 'mydb1.t1.i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7
-- !query
@@ -99,7 +99,7 @@ SELECT i1 FROM t1, mydb2.t1
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-Reference 'i1' is ambiguous, could be: mydb1.t1.i1, mydb2.t1.i1.; line 1 pos 7
+Reference 'i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb2.t1.i1.; line 1 pos 7
-- !query
@@ -108,7 +108,7 @@ SELECT t1.i1 FROM t1, mydb2.t1
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-Reference 't1.i1' is ambiguous, could be: mydb1.t1.i1, mydb2.t1.i1.; line 1 pos 7
+Reference 't1.i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb2.t1.i1.; line 1 pos 7
-- !query
@@ -125,7 +125,7 @@ SELECT i1 FROM t1, mydb1.t1
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-Reference 'i1' is ambiguous, could be: mydb2.t1.i1, mydb1.t1.i1.; line 1 pos 7
+Reference 'i1' is ambiguous, could be: spark_catalog.mydb2.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7
-- !query
@@ -134,7 +134,7 @@ SELECT t1.i1 FROM t1, mydb1.t1
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-Reference 't1.i1' is ambiguous, could be: mydb2.t1.i1, mydb1.t1.i1.; line 1 pos 7
+Reference 't1.i1' is ambiguous, could be: spark_catalog.mydb2.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7
-- !query
@@ -143,7 +143,7 @@ SELECT i1 FROM t1, mydb2.t1
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-Reference 'i1' is ambiguous, could be: mydb2.t1.i1, mydb2.t1.i1.; line 1 pos 7
+Reference 'i1' is ambiguous, could be: spark_catalog.mydb2.t1.i1, spark_catalog.mydb2.t1.i1.; line 1 pos 7
-- !query
@@ -152,7 +152,7 @@ SELECT t1.i1 FROM t1, mydb2.t1
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-Reference 't1.i1' is ambiguous, could be: mydb2.t1.i1, mydb2.t1.i1.; line 1 pos 7
+Reference 't1.i1' is ambiguous, could be: spark_catalog.mydb2.t1.i1, spark_catalog.mydb2.t1.i1.; line 1 pos 7
-- !query
@@ -161,7 +161,7 @@ SELECT db1.t1.i1 FROM t1, mydb2.t1
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-cannot resolve '`db1.t1.i1`' given input columns: [mydb2.t1.i1, mydb2.t1.i1]; line 1 pos 7
+cannot resolve '`db1.t1.i1`' given input columns: [spark_catalog.mydb2.t1.i1, spark_catalog.mydb2.t1.i1]; line 1 pos 7
-- !query
@@ -186,7 +186,7 @@ SELECT mydb1.t1 FROM t1
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-cannot resolve '`mydb1.t1`' given input columns: [mydb1.t1.i1]; line 1 pos 7
+cannot resolve '`mydb1.t1`' given input columns: [spark_catalog.mydb1.t1.i1]; line 1 pos 7
-- !query
@@ -204,7 +204,7 @@ SELECT t1 FROM mydb1.t1
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-cannot resolve '`t1`' given input columns: [mydb1.t1.i1]; line 1 pos 7
+cannot resolve '`t1`' given input columns: [spark_catalog.mydb1.t1.i1]; line 1 pos 7
-- !query
@@ -221,7 +221,7 @@ SELECT mydb1.t1.i1 FROM t1
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-cannot resolve '`mydb1.t1.i1`' given input columns: [mydb2.t1.i1]; line 1 pos 7
+cannot resolve '`mydb1.t1.i1`' given input columns: [spark_catalog.mydb2.t1.i1]; line 1 pos 7
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
index 85ce9786..1f2bd57 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
@@ -56,7 +56,7 @@ CREATE VIEW key_dependent_view AS
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-expression 'default.view_base_table.`data`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;
+expression 'spark_catalog.default.view_base_table.`data`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out
index 5332dff..20f4f6b 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out
@@ -536,7 +536,7 @@ SELECT '' AS `xxx`, i, k, t
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-Reference 'i' is ambiguous, could be: default.j1_tbl.i, default.j2_tbl.i.; line 1 pos 20
+Reference 'i' is ambiguous, could be: spark_catalog.default.j1_tbl.i, spark_catalog.default.j2_tbl.i.; line 1 pos 20
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out
index cbf4cfa..d8d33d9 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out
@@ -143,7 +143,7 @@ SELECT a FROM test_having HAVING min(a) < max(a)
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-grouping expressions sequence is empty, and 'default.test_having.`a`' is not an aggregate function. Wrap '(min(default.test_having.`a`) AS `min(a#x)`, max(default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'default.test_having.`a`' in first() (or first_value) if you don't care which value you get.;
+grouping expressions sequence is empty, and 'spark_catalog.default.test_having.`a`' is not an aggregate function. Wrap '(min(spark_catalog.default.test_having.`a`) AS `min(a#x)`, max(spark_catalog.default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'spark_catalog.default.test_having.`a`' in first() (or first_value) if you don't care which value you get.;
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out
index 5a52358..acce688 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out
@@ -244,7 +244,7 @@ from t1 where f1 = f2
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-cannot resolve '(PARTITION BY default.t1.`f1` RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING)' due to data type mismatch: A range window frame cannot be used in an unordered window specification.; line 1 pos 24
+cannot resolve '(PARTITION BY spark_catalog.default.t1.`f1` RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING)' due to data type mismatch: A range window frame cannot be used in an unordered window specification.; line 1 pos 24
-- !query
@@ -306,7 +306,7 @@ org.apache.spark.sql.AnalysisException
The query operator `Join` contains one or more unsupported
expression types Aggregate, Window or Generate.
-Invalid expressions: [row_number() OVER (ORDER BY default.empsalary.`salary` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)];
+Invalid expressions: [row_number() OVER (ORDER BY spark_catalog.default.empsalary.`salary` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)];
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out
index 3cc14ff..188b57f 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out
@@ -536,7 +536,7 @@ SELECT udf('') AS `xxx`, udf(i) AS i, udf(k), udf(t) AS t
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-Reference 'i' is ambiguous, could be: default.j1_tbl.i, default.j2_tbl.i.; line 1 pos 29
+Reference 'i' is ambiguous, could be: spark_catalog.default.j1_tbl.i, spark_catalog.default.j2_tbl.i.; line 1 pos 29
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out
index bb108a2..50b6e60 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out
@@ -143,7 +143,7 @@ SELECT udf(a) FROM test_having HAVING udf(min(a)) < udf(max(a))
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-grouping expressions sequence is empty, and 'default.test_having.`a`' is not an aggregate function. Wrap '(min(default.test_having.`a`) AS `min(a#x)`, max(default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'default.test_having.`a`' in first() (or first_value) if you don't care which value you get.;
+grouping expressions sequence is empty, and 'spark_catalog.default.test_having.`a`' is not an aggregate function. Wrap '(min(spark_catalog.default.test_having.`a`) AS `min(a#x)`, max(spark_catalog.default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'spark_catalog.default.test_having.`a`' in first() (or first_value) if you don't care which value you get.;
-- !query
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 563b4d1..81dfa798 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -2797,7 +2797,9 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
sql("SELECT * FROM t, S WHERE c = C")
}.message
assert(
- m.contains("cannot resolve '(default.t.`c` = default.S.`C`)' due to data type mismatch"))
+ m.contains(
+ "cannot resolve '(spark_catalog.default.t.`c` = spark_catalog.default.S.`C`)' " +
+ "due to data type mismatch"))
}
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index 1fc0bb1..ba4200d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -758,14 +758,10 @@ class DataSourceV2SQLSuite
checkAnswer(sql("select i from t"), Row(1))
checkAnswer(sql("select t.i from t"), Row(1))
checkAnswer(sql("select default.t.i from t"), Row(1))
+ checkAnswer(sql("select spark_catalog.default.t.i from t"), Row(1))
checkAnswer(sql("select t.i from spark_catalog.default.t"), Row(1))
checkAnswer(sql("select default.t.i from spark_catalog.default.t"), Row(1))
-
- // catalog name cannot be used for tables in the session catalog.
- val ex = intercept[AnalysisException] {
- sql(s"select spark_catalog.default.t.i from spark_catalog.default.t")
- }
- assert(ex.getMessage.contains("cannot resolve '`spark_catalog.default.t.i`"))
+ checkAnswer(sql("select spark_catalog.default.t.i from spark_catalog.default.t"), Row(1))
}
}
}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
index b8ef44b..9cd56f1 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
@@ -62,7 +62,7 @@ class HiveMetastoreCatalogSuite extends TestHiveSingleton with SQLTestUtils {
spark.sql("create view vw1 as select 1 as id")
val plan = spark.sql("select id from vw1").queryExecution.analyzed
val aliases = plan.collect {
- case x @ SubqueryAlias(AliasIdentifier("vw1", Seq("default")), _) => x
+ case x @ SubqueryAlias(AliasIdentifier("vw1", Seq("spark_catalog", "default")), _) => x
}
assert(aliases.size == 1)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org