You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2023/07/28 04:44:18 UTC
[spark] branch master updated: [SPARK-42098][SQL] Fix ResolveInlineTables can not handle with RuntimeReplaceable expression
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new f235c9f622d [SPARK-42098][SQL] Fix ResolveInlineTables can not handle with RuntimeReplaceable expression
f235c9f622d is described below
commit f235c9f622ddeb4c6a5cce8903130709dcd2217c
Author: Jia Fan <fa...@qq.com>
AuthorDate: Fri Jul 28 12:44:02 2023 +0800
[SPARK-42098][SQL] Fix ResolveInlineTables can not handle with RuntimeReplaceable expression
### What changes were proposed in this pull request?
Fix `ResolveInlineTables` can not handle with `RuntimeReplaceable` expression
eg:
```sql
select * from values (try_add(5, 0))
```
The error will throw:
```java
[INVALID_INLINE_TABLE.CANNOT_EVALUATE_EXPRESSION_IN_INLINE_TABLE] Invalid inline table. Cannot evaluate the expression "try_add(5, 0)" in inline table definition.; line 1 pos 22
```
### Why are the changes needed?
Fix the bug ResolveInlineTables can not handle with RuntimeReplaceable expression
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
add new test.
Closes #42110 from Hisoka-X/SPARK-42098_inline_table_replaceable.
Authored-by: Jia Fan <fa...@qq.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../catalyst/analysis/ResolveInlineTables.scala | 6 ++++--
.../analyzer-results/inline-table.sql.out | 21 +++++++++++++++++++
.../resources/sql-tests/inputs/inline-table.sql | 5 +++++
.../sql-tests/results/inline-table.sql.out | 24 ++++++++++++++++++++++
4 files changed, 54 insertions(+), 2 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTables.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTables.scala
index cf706171cd9..c203f9fa39d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTables.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTables.scala
@@ -21,6 +21,7 @@ import scala.util.control.NonFatal
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.AliasHelper
+import org.apache.spark.sql.catalyst.optimizer.ReplaceExpressions
import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.catalyst.trees.AlwaysProcess
@@ -36,8 +37,9 @@ object ResolveInlineTables extends Rule[LogicalPlan] with CastSupport with Alias
AlwaysProcess.fn, ruleId) {
case table: UnresolvedInlineTable if table.expressionsResolved =>
validateInputDimension(table)
- validateInputEvaluable(table)
- convert(table)
+ val newTable = ReplaceExpressions(table).asInstanceOf[UnresolvedInlineTable]
+ validateInputEvaluable(newTable)
+ convert(newTable)
}
/**
diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/inline-table.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/inline-table.sql.out
index f3e6eb4d8dc..2a17f092a06 100644
--- a/sql/core/src/test/resources/sql-tests/analyzer-results/inline-table.sql.out
+++ b/sql/core/src/test/resources/sql-tests/analyzer-results/inline-table.sql.out
@@ -220,3 +220,24 @@ select * from values (timestamp('1991-12-06 00:00:00.0'), array(timestamp('1991-
Project [a#x, b#x]
+- SubqueryAlias data
+- LocalRelation [a#x, b#x]
+
+
+-- !query
+select * from values (try_add(5, 0))
+-- !query analysis
+Project [col1#x]
++- LocalRelation [col1#x]
+
+
+-- !query
+select * from values (try_divide(5, 0))
+-- !query analysis
+Project [col1#x]
++- LocalRelation [col1#x]
+
+
+-- !query
+select * from values (10 + try_divide(5, 0))
+-- !query analysis
+Project [col1#x]
++- LocalRelation [col1#x]
diff --git a/sql/core/src/test/resources/sql-tests/inputs/inline-table.sql b/sql/core/src/test/resources/sql-tests/inputs/inline-table.sql
index fd8bb2d837d..6867248f576 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/inline-table.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/inline-table.sql
@@ -55,3 +55,8 @@ select * from values ("one", count(1)), ("two", 2) as data(a, b);
-- string to timestamp
select * from values (timestamp('1991-12-06 00:00:00.0'), array(timestamp('1991-12-06 01:00:00.0'), timestamp('1991-12-06 12:00:00.0'))) as data(a, b);
+
+-- ReplaceExpressions as row
+select * from values (try_add(5, 0));
+select * from values (try_divide(5, 0));
+select * from values (10 + try_divide(5, 0));
diff --git a/sql/core/src/test/resources/sql-tests/results/inline-table.sql.out b/sql/core/src/test/resources/sql-tests/results/inline-table.sql.out
index e735dd0dd5a..709d7ab73f6 100644
--- a/sql/core/src/test/resources/sql-tests/results/inline-table.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/inline-table.sql.out
@@ -242,3 +242,27 @@ select * from values (timestamp('1991-12-06 00:00:00.0'), array(timestamp('1991-
struct<a:timestamp,b:array<timestamp>>
-- !query output
1991-12-06 00:00:00 [1991-12-06 01:00:00,1991-12-06 12:00:00]
+
+
+-- !query
+select * from values (try_add(5, 0))
+-- !query schema
+struct<col1:int>
+-- !query output
+5
+
+
+-- !query
+select * from values (try_divide(5, 0))
+-- !query schema
+struct<col1:double>
+-- !query output
+NULL
+
+
+-- !query
+select * from values (10 + try_divide(5, 0))
+-- !query schema
+struct<col1:double>
+-- !query output
+NULL
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org