You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/04/07 13:56:54 UTC
(spark) branch master updated: [SPARK-51395][SQL] Refine handling of default values in procedures
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 738a50364f16 [SPARK-51395][SQL] Refine handling of default values in procedures
738a50364f16 is described below
commit 738a50364f16c579e370736df92bf01fe0a06af1
Author: Anton Okolnychyi <ao...@apache.org>
AuthorDate: Mon Apr 7 21:56:36 2025 +0800
[SPARK-51395][SQL] Refine handling of default values in procedures
### What changes were proposed in this pull request?
This PR refines handling of default values in procedures that will be released in 4.0.
### Why are the changes needed?
These changes are needed as connectors like Iceberg may not have utilities to generate SQL strings containing Spark SQL dialects. The API should be changed to allow either a DSv2 expression or a SQL string.
### Does this PR introduce _any_ user-facing change?
Yes, but the stored procedure API hasn't been released yet.
### How was this patch tested?
This PR comes with tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #50197 from aokolnychyi/spark-51395.
Authored-by: Anton Okolnychyi <ao...@apache.org>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../spark/sql/connector/catalog/DefaultValue.java | 95 +++++
.../catalog/procedures/ProcedureParameter.java | 30 +-
.../catalyst/expressions/V2ExpressionUtils.scala | 172 +++++++-
.../plans/logical/FunctionBuilderBase.scala | 6 +-
.../catalyst/util/ResolveDefaultColumnsUtil.scala | 46 ++-
.../connector/ProcedureParameterImpl.scala | 3 +-
.../spark/sql/connector/ProcedureSuite.scala | 45 ++-
.../datasources/v2/DataSourceV2StrategySuite.scala | 433 ++++++++++++++++++++-
8 files changed, 809 insertions(+), 21 deletions(-)
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DefaultValue.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DefaultValue.java
new file mode 100644
index 000000000000..6e487ce326a4
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DefaultValue.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog;
+
+import java.util.Map;
+import java.util.Objects;
+import javax.annotation.Nullable;
+
+import org.apache.spark.SparkIllegalArgumentException;
+import org.apache.spark.annotation.Evolving;
+import org.apache.spark.sql.connector.expressions.Expression;
+
+/**
+ * A class that represents default values.
+ * <p>
+ * Connectors can define default values using either a SQL string (Spark SQL dialect) or an
+ * {@link Expression expression} if the default value can be expressed as a supported connector
+ * expression. If both the SQL string and the expression are provided, Spark first attempts to
+ * convert the given expression to its internal representation. If the expression cannot be
+ * converted, and a SQL string is provided, Spark will fall back to parsing the SQL string.
+ *
+ * @since 4.1.0
+ */
+@Evolving
+public class DefaultValue {
+ private final String sql;
+ private final Expression expr;
+
+ public DefaultValue(String sql) {
+ this(sql, null /* no expression */);
+ }
+
+ public DefaultValue(Expression expr) {
+ this(null /* no sql */, expr);
+ }
+
+ public DefaultValue(String sql, Expression expr) {
+ if (sql == null && expr == null) {
+ throw new SparkIllegalArgumentException(
+ "INTERNAL_ERROR",
+ Map.of("message", "SQL and expression can't be both null"));
+ }
+ this.sql = sql;
+ this.expr = expr;
+ }
+
+ /**
+ * Returns the SQL representation of the default value (Spark SQL dialect), if provided.
+ */
+ @Nullable
+ public String getSql() {
+ return sql;
+ }
+
+ /**
+ * Returns the expression representing the default value, if provided.
+ */
+ @Nullable
+ public Expression getExpression() {
+ return expr;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) return true;
+ if (other == null || getClass() != other.getClass()) return false;
+ DefaultValue that = (DefaultValue) other;
+ return Objects.equals(sql, that.sql) && Objects.equals(expr, that.expr);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(sql, expr);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("DefaultValue{sql=%s, expression=%s}", sql, expr);
+ }
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/procedures/ProcedureParameter.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/procedures/ProcedureParameter.java
index 18c76833c587..3d837be366f7 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/procedures/ProcedureParameter.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/procedures/ProcedureParameter.java
@@ -20,6 +20,8 @@ package org.apache.spark.sql.connector.catalog.procedures;
import javax.annotation.Nullable;
import org.apache.spark.annotation.Evolving;
+import org.apache.spark.sql.connector.catalog.DefaultValue;
+import org.apache.spark.sql.connector.expressions.Expression;
import org.apache.spark.sql.internal.connector.ProcedureParameterImpl;
import org.apache.spark.sql.types.DataType;
@@ -68,7 +70,7 @@ public interface ProcedureParameter {
* null if not provided.
*/
@Nullable
- String defaultValueExpression();
+ DefaultValue defaultValue();
/**
* Returns the comment of this parameter or null if not provided.
@@ -89,7 +91,7 @@ public interface ProcedureParameter {
private final Mode mode;
private final String name;
private final DataType dataType;
- private String defaultValueExpression;
+ private DefaultValue defaultValue;
private String comment;
private Builder(Mode mode, String name, DataType dataType) {
@@ -99,10 +101,26 @@ public interface ProcedureParameter {
}
/**
- * Sets the default value expression of the parameter.
+ * Sets the default value of the parameter using SQL.
*/
- public Builder defaultValue(String defaultValueExpression) {
- this.defaultValueExpression = defaultValueExpression;
+ public Builder defaultValue(String sql) {
+ this.defaultValue = new DefaultValue(sql);
+ return this;
+ }
+
+ /**
+ * Sets the default value of the parameter using an expression.
+ */
+ public Builder defaultValue(Expression expression) {
+ this.defaultValue = new DefaultValue(expression);
+ return this;
+ }
+
+ /**
+ * Sets the default value of the parameter.
+ */
+ public Builder defaultValue(DefaultValue defaultValue) {
+ this.defaultValue = defaultValue;
return this;
}
@@ -118,7 +136,7 @@ public interface ProcedureParameter {
* Builds the stored procedure parameter.
*/
public ProcedureParameter build() {
- return new ProcedureParameterImpl(mode, name, dataType, defaultValueExpression, comment);
+ return new ProcedureParameterImpl(mode, name, dataType, defaultValue, comment);
}
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtils.scala
index d14c8cb67538..7cc03f3ac3fa 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtils.scala
@@ -23,14 +23,15 @@ import org.apache.spark.internal.{Logging, MDC}
import org.apache.spark.internal.LogKeys.{FUNCTION_NAME, FUNCTION_PARAM}
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.{InternalRow, SQLConfHelper}
-import org.apache.spark.sql.catalyst.analysis.NoSuchFunctionException
+import org.apache.spark.sql.catalyst.analysis.{NoSuchFunctionException, UnresolvedAttribute}
import org.apache.spark.sql.catalyst.encoders.EncoderUtils
import org.apache.spark.sql.catalyst.expressions.objects.{Invoke, StaticInvoke}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.connector.catalog.{FunctionCatalog, Identifier}
import org.apache.spark.sql.connector.catalog.functions._
import org.apache.spark.sql.connector.catalog.functions.ScalarFunction.MAGIC_METHOD_NAME
-import org.apache.spark.sql.connector.expressions.{BucketTransform, Expression => V2Expression, FieldReference, IdentityTransform, Literal => V2Literal, NamedReference, NamedTransform, NullOrdering => V2NullOrdering, SortDirection => V2SortDirection, SortOrder => V2SortOrder, SortValue, Transform}
+import org.apache.spark.sql.connector.expressions.{BucketTransform, Cast => V2Cast, Expression => V2Expression, FieldReference, GeneralScalarExpression, IdentityTransform, Literal => V2Literal, NamedReference, NamedTransform, NullOrdering => V2NullOrdering, SortDirection => V2SortDirection, SortOrder => V2SortOrder, SortValue, Transform}
+import org.apache.spark.sql.connector.expressions.filter.{AlwaysFalse, AlwaysTrue}
import org.apache.spark.sql.errors.DataTypeErrors.toSQLId
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.types._
@@ -205,4 +206,171 @@ object V2ExpressionUtils extends SQLConfHelper with Logging {
None
}
}
+
+ def toCatalyst(expr: V2Expression): Option[Expression] = expr match {
+ case _: AlwaysTrue => Some(Literal.TrueLiteral)
+ case _: AlwaysFalse => Some(Literal.FalseLiteral)
+ case l: V2Literal[_] => Some(Literal(l.value, l.dataType))
+ case r: NamedReference => Some(UnresolvedAttribute(r.fieldNames.toImmutableArraySeq))
+ case c: V2Cast => toCatalyst(c.expression).map(Cast(_, c.dataType, ansiEnabled = true))
+ case e: GeneralScalarExpression => convertScalarExpr(e)
+ case _ => None
+ }
+
+ private def convertScalarExpr(expr: GeneralScalarExpression): Option[Expression] = {
+ convertPredicate(expr)
+ .orElse(convertConditionalFunc(expr))
+ .orElse(convertMathFunc(expr))
+ .orElse(convertBitwiseFunc(expr))
+ .orElse(convertTrigonometricFunc(expr))
+ }
+
+ private def convertPredicate(expr: GeneralScalarExpression): Option[Expression] = {
+ expr.name match {
+ case "IS_NULL" => convertUnaryExpr(expr, IsNull)
+ case "IS_NOT_NULL" => convertUnaryExpr(expr, IsNotNull)
+ case "NOT" => convertUnaryExpr(expr, Not)
+ case "=" => convertBinaryExpr(expr, EqualTo)
+ case "<=>" => convertBinaryExpr(expr, EqualNullSafe)
+ case ">" => convertBinaryExpr(expr, GreaterThan)
+ case ">=" => convertBinaryExpr(expr, GreaterThanOrEqual)
+ case "<" => convertBinaryExpr(expr, LessThan)
+ case "<=" => convertBinaryExpr(expr, LessThanOrEqual)
+ case "<>" => convertBinaryExpr(expr, (left, right) => Not(EqualTo(left, right)))
+ case "AND" => convertBinaryExpr(expr, And)
+ case "OR" => convertBinaryExpr(expr, Or)
+ case "STARTS_WITH" => convertBinaryExpr(expr, StartsWith)
+ case "ENDS_WITH" => convertBinaryExpr(expr, EndsWith)
+ case "CONTAINS" => convertBinaryExpr(expr, Contains)
+ case "IN" => convertExpr(expr, children => In(children.head, children.tail))
+ case _ => None
+ }
+ }
+
+ private def convertConditionalFunc(expr: GeneralScalarExpression): Option[Expression] = {
+ expr.name match {
+ case "CASE_WHEN" =>
+ convertExpr(expr, children =>
+ if (children.length % 2 == 0) {
+ val branches = children.grouped(2).map { case Seq(c, v) => (c, v) }.toSeq
+ CaseWhen(branches, None)
+ } else {
+ val (pairs, last) = children.splitAt(children.length - 1)
+ val branches = pairs.grouped(2).map { case Seq(c, v) => (c, v) }.toSeq
+ CaseWhen(branches, Some(last.head))
+ })
+ case _ => None
+ }
+ }
+
+ private def convertMathFunc(expr: GeneralScalarExpression): Option[Expression] = {
+ expr.name match {
+ case "+" => convertBinaryExpr(expr, Add(_, _, evalMode = EvalMode.ANSI))
+ case "-" =>
+ if (expr.children.length == 1) {
+ convertUnaryExpr(expr, UnaryMinus(_, failOnError = true))
+ } else if (expr.children.length == 2) {
+ convertBinaryExpr(expr, Subtract(_, _, evalMode = EvalMode.ANSI))
+ } else {
+ None
+ }
+ case "*" => convertBinaryExpr(expr, Multiply(_, _, evalMode = EvalMode.ANSI))
+ case "/" => convertBinaryExpr(expr, Divide(_, _, evalMode = EvalMode.ANSI))
+ case "%" => convertBinaryExpr(expr, Remainder(_, _, evalMode = EvalMode.ANSI))
+ case "ABS" => convertUnaryExpr(expr, Abs(_, failOnError = true))
+ case "COALESCE" => convertExpr(expr, Coalesce)
+ case "GREATEST" => convertExpr(expr, Greatest)
+ case "LEAST" => convertExpr(expr, Least)
+ case "RAND" =>
+ if (expr.children.isEmpty) {
+ Some(new Rand())
+ } else if (expr.children.length == 1) {
+ convertUnaryExpr(expr, new Rand(_))
+ } else {
+ None
+ }
+ case "LOG" => convertBinaryExpr(expr, Logarithm)
+ case "LOG10" => convertUnaryExpr(expr, Log10)
+ case "LOG2" => convertUnaryExpr(expr, Log2)
+ case "LN" => convertUnaryExpr(expr, Log)
+ case "EXP" => convertUnaryExpr(expr, Exp)
+ case "POWER" => convertBinaryExpr(expr, Pow)
+ case "SQRT" => convertUnaryExpr(expr, Sqrt)
+ case "FLOOR" => convertUnaryExpr(expr, Floor)
+ case "CEIL" => convertUnaryExpr(expr, Ceil)
+ case "ROUND" => convertBinaryExpr(expr, Round(_, _, ansiEnabled = true))
+ case "CBRT" => convertUnaryExpr(expr, Cbrt)
+ case "DEGREES" => convertUnaryExpr(expr, ToDegrees)
+ case "RADIANS" => convertUnaryExpr(expr, ToRadians)
+ case "SIGN" => convertUnaryExpr(expr, Signum)
+ case "WIDTH_BUCKET" =>
+ convertExpr(
+ expr,
+ children => WidthBucket(children(0), children(1), children(2), children(3)))
+ case _ => None
+ }
+ }
+
+ private def convertTrigonometricFunc(expr: GeneralScalarExpression): Option[Expression] = {
+ expr.name match {
+ case "SIN" => convertUnaryExpr(expr, Sin)
+ case "SINH" => convertUnaryExpr(expr, Sinh)
+ case "COS" => convertUnaryExpr(expr, Cos)
+ case "COSH" => convertUnaryExpr(expr, Cosh)
+ case "TAN" => convertUnaryExpr(expr, Tan)
+ case "TANH" => convertUnaryExpr(expr, Tanh)
+ case "COT" => convertUnaryExpr(expr, Cot)
+ case "ASIN" => convertUnaryExpr(expr, Asin)
+ case "ASINH" => convertUnaryExpr(expr, Asinh)
+ case "ACOS" => convertUnaryExpr(expr, Acos)
+ case "ACOSH" => convertUnaryExpr(expr, Acosh)
+ case "ATAN" => convertUnaryExpr(expr, Atan)
+ case "ATANH" => convertUnaryExpr(expr, Atanh)
+ case "ATAN2" => convertBinaryExpr(expr, Atan2)
+ case _ => None
+ }
+ }
+
+ private def convertBitwiseFunc(expr: GeneralScalarExpression): Option[Expression] = {
+ expr.name match {
+ case "~" => convertUnaryExpr(expr, BitwiseNot)
+ case "&" => convertBinaryExpr(expr, BitwiseAnd)
+ case "|" => convertBinaryExpr(expr, BitwiseOr)
+ case "^" => convertBinaryExpr(expr, BitwiseXor)
+ case _ => None
+ }
+ }
+
+ private def convertUnaryExpr(
+ expr: GeneralScalarExpression,
+ catalystExprBuilder: Expression => Expression): Option[Expression] = {
+ expr.children match {
+ case Array(child) => toCatalyst(child).map(catalystExprBuilder)
+ case _ => None
+ }
+ }
+
+ private def convertBinaryExpr(
+ expr: GeneralScalarExpression,
+ catalystExprBuilder: (Expression, Expression) => Expression): Option[Expression] = {
+ expr.children match {
+ case Array(left, right) =>
+ for {
+ catalystLeft <- toCatalyst(left)
+ catalystRight <- toCatalyst(right)
+ } yield catalystExprBuilder(catalystLeft, catalystRight)
+ case _ => None
+ }
+ }
+
+ private def convertExpr(
+ expr: GeneralScalarExpression,
+ catalystExprBuilder: Seq[Expression] => Expression): Option[Expression] = {
+ val catalystChildren = expr.children.flatMap(toCatalyst).toImmutableArraySeq
+ if (expr.children.length == catalystChildren.length) {
+ Some(catalystExprBuilder(catalystChildren))
+ } else {
+ None
+ }
+ }
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/FunctionBuilderBase.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/FunctionBuilderBase.scala
index 75b2fcd3a5f3..638d20cff928 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/FunctionBuilderBase.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/FunctionBuilderBase.scala
@@ -135,10 +135,10 @@ object NamedParametersSupport {
}
private def toInputParameter(param: ProcedureParameter): InputParameter = {
- val defaultValue = Option(param.defaultValueExpression).map { expr =>
- ResolveDefaultColumns.analyze(param.name, param.dataType, expr, "CALL")
+ val defaultValueExpr = Option(param.defaultValue).map { defaultValue =>
+ ResolveDefaultColumns.analyze(param.name, param.dataType, defaultValue, "CALL")
}
- InputParameter(param.name, defaultValue)
+ InputParameter(param.name, defaultValueExpr)
}
private def defaultRearrange(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala
index 61c6ae448c77..2dca2eed8573 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala
@@ -32,7 +32,7 @@ import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding, Optimizer}
import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParseException}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.trees.TreePattern.PLAN_EXPRESSION
-import org.apache.spark.sql.connector.catalog.{CatalogManager, FunctionCatalog, Identifier, TableCatalog, TableCatalogCapability}
+import org.apache.spark.sql.connector.catalog.{CatalogManager, DefaultValue, FunctionCatalog, Identifier, TableCatalog, TableCatalogCapability}
import org.apache.spark.sql.connector.catalog.functions.UnboundFunction
import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryErrorsBase}
import org.apache.spark.sql.internal.SQLConf
@@ -284,8 +284,48 @@ object ResolveDefaultColumns extends QueryErrorsBase
throw QueryCompilationErrors.defaultValuesUnresolvedExprError(
statementType, colName, defaultSQL, ex)
}
+ analyze(colName, dataType, parsed, defaultSQL, statementType)
+ }
+
+ /**
+ * Analyzes the connector default value.
+ *
+ * If the default value is defined as a connector expression, Spark first attempts to convert it
+ * to a Catalyst expression. If conversion fails but a SQL string is provided, the SQL is parsed
+ * instead. If only a SQL string is present, it is parsed directly.
+ *
+ * @return the result of the analysis and constant-folding operation
+ */
+ def analyze(
+ colName: String,
+ dataType: DataType,
+ defaultValue: DefaultValue,
+ statementType: String): Expression = {
+ if (defaultValue.getExpression != null) {
+ V2ExpressionUtils.toCatalyst(defaultValue.getExpression) match {
+ case Some(defaultExpr) =>
+ val defaultSQL = Option(defaultValue.getSql).getOrElse(defaultExpr.sql)
+ analyze(colName, dataType, defaultExpr, defaultSQL, statementType)
+
+ case None if defaultValue.getSql != null =>
+ analyze(colName, dataType, defaultValue.getSql, statementType)
+
+ case _ =>
+ throw SparkException.internalError(s"Can't convert $defaultValue to Catalyst")
+ }
+ } else {
+ analyze(colName, dataType, defaultValue.getSql, statementType)
+ }
+ }
+
+ private def analyze(
+ colName: String,
+ dataType: DataType,
+ defaultExpr: Expression,
+ defaultSQL: String,
+ statementType: String): Expression = {
// Check invariants before moving on to analysis.
- if (parsed.containsPattern(PLAN_EXPRESSION)) {
+ if (defaultExpr.containsPattern(PLAN_EXPRESSION)) {
throw QueryCompilationErrors.defaultValuesMayNotContainSubQueryExpressions(
statementType, colName, defaultSQL)
}
@@ -293,7 +333,7 @@ object ResolveDefaultColumns extends QueryErrorsBase
// Analyze the parse result.
val plan = try {
val analyzer: Analyzer = DefaultColumnAnalyzer
- val analyzed = analyzer.execute(Project(Seq(Alias(parsed, colName)()), OneRowRelation()))
+ val analyzed = analyzer.execute(Project(Seq(Alias(defaultExpr, colName)()), OneRowRelation()))
analyzer.checkAnalysis(analyzed)
// Eagerly execute finish-analysis and constant-folding rules before checking whether the
// expression is foldable and resolved.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/ProcedureParameterImpl.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/ProcedureParameterImpl.scala
index 01ea48af1537..ede9c0915ef2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/ProcedureParameterImpl.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/ProcedureParameterImpl.scala
@@ -17,6 +17,7 @@
package org.apache.spark.sql.internal.connector
+import org.apache.spark.sql.connector.catalog.DefaultValue
import org.apache.spark.sql.connector.catalog.procedures.ProcedureParameter
import org.apache.spark.sql.connector.catalog.procedures.ProcedureParameter.Mode
import org.apache.spark.sql.types.DataType
@@ -25,5 +26,5 @@ case class ProcedureParameterImpl(
mode: Mode,
name: String,
dataType: DataType,
- defaultValueExpression: String,
+ defaultValue: DefaultValue,
comment: String) extends ProcedureParameter
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/ProcedureSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/ProcedureSuite.scala
index 03f83e996d20..a221fd16d3c5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/ProcedureSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/ProcedureSuite.scala
@@ -25,15 +25,16 @@ import org.apache.spark.{SPARK_DOC_ROOT, SparkException, SparkNumberFormatExcept
import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.util.TypeUtils.toSQLId
-import org.apache.spark.sql.connector.catalog.{BasicInMemoryTableCatalog, Identifier, InMemoryCatalog}
+import org.apache.spark.sql.connector.catalog.{BasicInMemoryTableCatalog, DefaultValue, Identifier, InMemoryCatalog}
import org.apache.spark.sql.connector.catalog.procedures.{BoundProcedure, ProcedureParameter, UnboundProcedure}
import org.apache.spark.sql.connector.catalog.procedures.ProcedureParameter.Mode
import org.apache.spark.sql.connector.catalog.procedures.ProcedureParameter.Mode.{IN, INOUT, OUT}
+import org.apache.spark.sql.connector.expressions.{Expression, GeneralScalarExpression, LiteralValue}
import org.apache.spark.sql.connector.read.{LocalScan, Scan}
import org.apache.spark.sql.errors.DataTypeErrors.{toSQLType, toSQLValue}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types.{DataType, DataTypes, StructField, StructType}
+import org.apache.spark.sql.types.{DataType, DataTypes, IntegerType, StructField, StructType}
import org.apache.spark.unsafe.types.UTF8String
class ProcedureSuite extends QueryTest with SharedSparkSession with BeforeAndAfter {
@@ -448,6 +449,11 @@ class ProcedureSuite extends QueryTest with SharedSparkSession with BeforeAndAft
}
}
+ test("default values with expressions") {
+ catalog.createProcedure(Identifier.of(Array("ns"), "sum"), UnboundSumWithDefaultExpr)
+ checkAnswer(sql("CALL cat.ns.sum(5)"), Row(9) :: Nil)
+ }
+
object UnboundVoidProcedure extends UnboundProcedure {
override def name: String = "void"
override def description: String = "void procedure"
@@ -747,13 +753,46 @@ class ProcedureSuite extends QueryTest with SharedSparkSession with BeforeAndAft
}
}
+ object UnboundSumWithDefaultExpr extends UnboundProcedure {
+ override def name: String = "sum"
+ override def description: String = "sum longs"
+ override def bind(inputType: StructType): BoundProcedure = SumWithDefaultExpr
+ }
+
+ object SumWithDefaultExpr extends BoundProcedure {
+ override def name: String = "sum"
+
+ override def description: String = "sum longs"
+
+ override def isDeterministic: Boolean = true
+
+ override def parameters: Array[ProcedureParameter] = Array(
+ ProcedureParameter.in("in1", DataTypes.LongType).build(),
+ ProcedureParameter.in("in2", DataTypes.LongType)
+ .defaultValue(
+ new GeneralScalarExpression(
+ "+",
+ Array[Expression](LiteralValue(1, IntegerType), LiteralValue(3, IntegerType))))
+ .build()
+ )
+
+ def outputType: StructType = new StructType().add("out", DataTypes.LongType)
+
+ override def call(input: InternalRow): java.util.Iterator[Scan] = {
+ val in1 = input.getLong(0)
+ val in2 = input.getLong(1)
+ val result = Result(outputType, Array(InternalRow(in1 + in2)))
+ Collections.singleton[Scan](result).iterator()
+ }
+ }
+
case class Result(readSchema: StructType, rows: Array[InternalRow]) extends LocalScan
case class CustomParameterImpl(
mode: Mode,
name: String,
dataType: DataType) extends ProcedureParameter {
- override def defaultValueExpression: String = null
+ override def defaultValue: DefaultValue = null
override def comment: String = null
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala
index 3c4f5814375d..98a20e668588 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala
@@ -17,13 +17,15 @@
package org.apache.spark.sql.execution.datasources.v2
+import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.PlanTest
-import org.apache.spark.sql.connector.expressions.{FieldReference, LiteralValue}
-import org.apache.spark.sql.connector.expressions.filter.{And => V2And, Not => V2Not, Or => V2Or, Predicate}
+import org.apache.spark.sql.catalyst.util.V2ExpressionBuilder
+import org.apache.spark.sql.connector.expressions.{Expression => V2Expression, FieldReference, GeneralScalarExpression, LiteralValue}
+import org.apache.spark.sql.connector.expressions.filter.{AlwaysFalse, AlwaysTrue, And => V2And, Not => V2Not, Or => V2Or, Predicate}
import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types.{BooleanType, IntegerType, StringType, StructField, StructType}
+import org.apache.spark.sql.types.{BooleanType, DoubleType, IntegerType, LongType, StringType, StructField, StructType}
import org.apache.spark.unsafe.types.UTF8String
class DataSourceV2StrategySuite extends PlanTest with SharedSparkSession {
@@ -317,6 +319,401 @@ class DataSourceV2StrategySuite extends PlanTest with SharedSparkSession {
Some(new Predicate("=", Array(FieldReference("col"), LiteralValue(true, BooleanType)))))
}
+ test("inability to convert unknown expressions and predicates") {
+ val unknownExpr = new GeneralScalarExpression("UNKNOWN", Array())
+ assert(V2ExpressionUtils.toCatalyst(unknownExpr).isEmpty)
+
+ val unknownPred = new Predicate("UNKNOWN", Array())
+ assert(V2ExpressionUtils.toCatalyst(unknownPred).isEmpty)
+ }
+
+ test("round trip conversion of CASE_WHEN expression") {
+ // CASE WHEN cond1 THEN value1 WHEN cond2 THEN value2
+ checkRoundTripConversion(
+ catalystExpr = CaseWhen(
+ Seq(
+ (EqualTo(Literal(1), Literal(2)), Literal("a")),
+ (EqualTo(Literal(3), Literal(4)), Literal("b"))),
+ None),
+ v2Expr = new GeneralScalarExpression(
+ "CASE_WHEN",
+ Array(
+ new Predicate("=", Array(LiteralValue(1, IntegerType), LiteralValue(2, IntegerType))),
+ LiteralValue(UTF8String.fromString("a"), StringType),
+ new Predicate("=", Array(LiteralValue(3, IntegerType), LiteralValue(4, IntegerType))),
+ LiteralValue(UTF8String.fromString("b"), StringType))))
+
+ // CASE WHEN cond1 THEN value1 ELSE elseValue
+ checkRoundTripConversion(
+ catalystExpr = CaseWhen(
+ Seq((EqualTo(Literal(1), Literal(2)), Literal("yes"))),
+ Some(Literal("no"))),
+ v2Expr = new GeneralScalarExpression(
+ "CASE_WHEN",
+ Array(
+ new Predicate("=", Array(LiteralValue(1, IntegerType), LiteralValue(2, IntegerType))),
+ LiteralValue(UTF8String.fromString("yes"), StringType),
+ LiteralValue(UTF8String.fromString("no"), StringType))))
+
+ // CASE WHEN cond1 THEN true ELSE false
+ checkRoundTripConversion(
+ catalystExpr = CaseWhen(
+ Seq((EqualTo(Literal(1), Literal(2)), Literal(true))),
+ Some(Literal(false))),
+ v2Expr = new Predicate(
+ "CASE_WHEN",
+ Array(
+ new Predicate("=", Array(LiteralValue(1, IntegerType), LiteralValue(2, IntegerType))),
+ new AlwaysTrue,
+ new AlwaysFalse)),
+ isPredicate = true)
+
+ // CASE WHEN cond1 THEN true WHEN cond2 THEN false ELSE true
+ checkRoundTripConversion(
+ catalystExpr = CaseWhen(
+ Seq(
+ (EqualTo(Literal(1), Literal(2)), Literal(true)),
+ (EqualTo(Literal(3), Literal(4)), Literal(false))),
+ Some(Literal(true))),
+ v2Expr = new Predicate(
+ "CASE_WHEN",
+ Array(
+ new Predicate("=", Array(LiteralValue(1, IntegerType), LiteralValue(2, IntegerType))),
+ new AlwaysTrue,
+ new Predicate("=", Array(LiteralValue(3, IntegerType), LiteralValue(4, IntegerType))),
+ new AlwaysFalse,
+ new AlwaysTrue)),
+ isPredicate = true)
+ }
+
+ test("round trip conversion of math functions") {
+ checkRoundTripConversion(
+ catalystExpr = Log10(Literal(100)),
+ v2Expr = new GeneralScalarExpression("LOG10", Array(LiteralValue(100, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = new Rand(),
+ v2Expr = new GeneralScalarExpression("RAND", Array()))
+
+ checkRoundTripConversion(
+ catalystExpr = new Rand(Literal(17L)),
+ v2Expr = new GeneralScalarExpression("RAND", Array(LiteralValue(17L, LongType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Abs(Literal(-5)),
+ v2Expr = new GeneralScalarExpression("ABS", Array(LiteralValue(-5, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = UnaryMinus(Literal(5), failOnError = true),
+ v2Expr = new GeneralScalarExpression("-", Array(LiteralValue(5, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Log2(Literal(8)),
+ v2Expr = new GeneralScalarExpression("LOG2", Array(LiteralValue(8, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Log(Literal(100)),
+ v2Expr = new GeneralScalarExpression("LN", Array(LiteralValue(100, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Exp(Literal(1.0)),
+ v2Expr = new GeneralScalarExpression("EXP", Array(LiteralValue(1.0, DoubleType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Sqrt(Literal(4.0)),
+ v2Expr = new GeneralScalarExpression("SQRT", Array(LiteralValue(4.0, DoubleType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Floor(Literal(4.7)),
+ v2Expr = new GeneralScalarExpression("FLOOR", Array(LiteralValue(4.7, DoubleType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Ceil(Literal(4.3)),
+ v2Expr = new GeneralScalarExpression("CEIL", Array(LiteralValue(4.3, DoubleType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Sin(Literal(0)),
+ v2Expr = new GeneralScalarExpression("SIN", Array(LiteralValue(0, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Sinh(Literal(0)),
+ v2Expr = new GeneralScalarExpression("SINH", Array(LiteralValue(0, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Cos(Literal(0)),
+ v2Expr = new GeneralScalarExpression("COS", Array(LiteralValue(0, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Cosh(Literal(0)),
+ v2Expr = new GeneralScalarExpression("COSH", Array(LiteralValue(0, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Tan(Literal(0)),
+ v2Expr = new GeneralScalarExpression("TAN", Array(LiteralValue(0, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Tanh(Literal(0)),
+ v2Expr = new GeneralScalarExpression("TANH", Array(LiteralValue(0, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Cot(Literal(1)),
+ v2Expr = new GeneralScalarExpression("COT", Array(LiteralValue(1, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Asin(Literal(1.0)),
+ v2Expr = new GeneralScalarExpression("ASIN", Array(LiteralValue(1.0, DoubleType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Asinh(Literal(1.0)),
+ v2Expr = new GeneralScalarExpression("ASINH", Array(LiteralValue(1.0, DoubleType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Acos(Literal(1.0)),
+ v2Expr = new GeneralScalarExpression("ACOS", Array(LiteralValue(1.0, DoubleType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Acosh(Literal(1.0)),
+ v2Expr = new GeneralScalarExpression("ACOSH", Array(LiteralValue(1.0, DoubleType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Atan(Literal(1.0)),
+ v2Expr = new GeneralScalarExpression("ATAN", Array(LiteralValue(1.0, DoubleType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Atanh(Literal(0.5)),
+ v2Expr = new GeneralScalarExpression("ATANH", Array(LiteralValue(0.5, DoubleType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Cbrt(Literal(8.0)),
+ v2Expr = new GeneralScalarExpression("CBRT", Array(LiteralValue(8.0, DoubleType))))
+
+ checkRoundTripConversion(
+ catalystExpr = ToDegrees(Literal(3.14)),
+ v2Expr = new GeneralScalarExpression("DEGREES", Array(LiteralValue(3.14, DoubleType))))
+
+ checkRoundTripConversion(
+ catalystExpr = ToRadians(Literal(180.0)),
+ v2Expr = new GeneralScalarExpression("RADIANS", Array(LiteralValue(180.0, DoubleType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Signum(Literal(-42)),
+ v2Expr = new GeneralScalarExpression("SIGN", Array(LiteralValue(-42, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Add(Literal(1), Literal(2), EvalMode.ANSI),
+ v2Expr = new GeneralScalarExpression(
+ "+",
+ Array(LiteralValue(1, IntegerType), LiteralValue(2, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Subtract(Literal(5), Literal(3), EvalMode.ANSI),
+ v2Expr = new GeneralScalarExpression(
+ "-",
+ Array(LiteralValue(5, IntegerType), LiteralValue(3, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Multiply(Literal(2), Literal(4), EvalMode.ANSI),
+ v2Expr = new GeneralScalarExpression(
+ "*",
+ Array(LiteralValue(2, IntegerType), LiteralValue(4, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Divide(Literal(10), Literal(2), EvalMode.ANSI),
+ v2Expr = new GeneralScalarExpression(
+ "/",
+ Array(LiteralValue(10, IntegerType), LiteralValue(2, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Remainder(Literal(7), Literal(3), EvalMode.ANSI),
+ v2Expr = new GeneralScalarExpression(
+ "%",
+ Array(LiteralValue(7, IntegerType), LiteralValue(3, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Logarithm(Literal(10), Literal(100)),
+ v2Expr = new GeneralScalarExpression(
+ "LOG",
+ Array(LiteralValue(10, IntegerType), LiteralValue(100, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Pow(Literal(2), Literal(3)),
+ v2Expr = new GeneralScalarExpression(
+ "POWER",
+ Array(LiteralValue(2, IntegerType), LiteralValue(3, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Round(Literal(123.456), Literal(2), ansiEnabled = true),
+ v2Expr = new GeneralScalarExpression(
+ "ROUND",
+ Array(LiteralValue(123.456, DoubleType), LiteralValue(2, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Atan2(Literal(1.0), Literal(1.0)),
+ v2Expr = new GeneralScalarExpression(
+ "ATAN2",
+ Array(LiteralValue(1.0, DoubleType), LiteralValue(1.0, DoubleType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Coalesce(Seq(Literal(null, IntegerType), Literal(5))),
+ v2Expr = new GeneralScalarExpression(
+ "COALESCE",
+ Array(LiteralValue(null, IntegerType), LiteralValue(5, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Greatest(Seq(Literal(1), Literal(2))),
+ v2Expr = new GeneralScalarExpression(
+ "GREATEST",
+ Array(LiteralValue(1, IntegerType), LiteralValue(2, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Least(Seq(Literal(1), Literal(2))),
+ v2Expr = new GeneralScalarExpression(
+ "LEAST",
+ Array(LiteralValue(1, IntegerType), LiteralValue(2, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = WidthBucket(Literal(5), Literal(0), Literal(10), Literal(5)),
+ v2Expr = new GeneralScalarExpression(
+ "WIDTH_BUCKET",
+ Array(
+ LiteralValue(5, IntegerType),
+ LiteralValue(0, IntegerType),
+ LiteralValue(10, IntegerType),
+ LiteralValue(5, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Sqrt(Pow(Abs(Literal(-3)), Literal(2))),
+ v2Expr = new GeneralScalarExpression(
+ "SQRT",
+ Array(
+ new GeneralScalarExpression(
+ "POWER",
+ Array(new GeneralScalarExpression("ABS", Array(LiteralValue(-3, IntegerType))),
+ LiteralValue(2, IntegerType))))))
+ }
+
+ test("round trip conversion of bitwise functions") {
+ checkRoundTripConversion(
+ catalystExpr = BitwiseNot(Literal(5)),
+ v2Expr = new GeneralScalarExpression("~", Array(LiteralValue(5, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = BitwiseAnd(Literal(6), Literal(3)),
+ v2Expr = new GeneralScalarExpression("&", Array(
+ LiteralValue(6, IntegerType),
+ LiteralValue(3, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = BitwiseOr(Literal(4), Literal(1)),
+ v2Expr = new GeneralScalarExpression("|", Array(
+ LiteralValue(4, IntegerType),
+ LiteralValue(1, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = BitwiseXor(Literal(12), Literal(5)),
+ v2Expr = new GeneralScalarExpression("^", Array(
+ LiteralValue(12, IntegerType),
+ LiteralValue(5, IntegerType))))
+ }
+
+ test("round trip conversion of predicate expressions") {
+ checkRoundTripConversion(
+ catalystExpr = IsNull($"a".boolean),
+ v2Expr = new Predicate("IS_NULL", Array(FieldReference("a"))))
+
+ checkRoundTripConversion(
+ catalystExpr = IsNotNull($"a".boolean),
+ v2Expr = new Predicate("IS_NOT_NULL", Array(FieldReference("a"))))
+
+ checkV2Conversion(
+ catalystExpr = Not($"a".boolean),
+ v2Expr = new V2Not(new Predicate(
+ "=",
+ Array(FieldReference("a"), LiteralValue(true, BooleanType)))))
+
+ checkCatalystConversion(
+ v2Expr = new V2Not(new Predicate(
+ "=",
+ Array(FieldReference("a"), LiteralValue(true, BooleanType)))),
+ catalystExpr = Not(EqualTo($"a".boolean, Literal(true))))
+
+ checkRoundTripConversion(
+ catalystExpr = EqualTo(Literal(1), Literal(2)),
+ v2Expr = new Predicate(
+ "=",
+ Array(LiteralValue(1, IntegerType), LiteralValue(2, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = EqualNullSafe(Literal(1), Literal(2)),
+ v2Expr = new Predicate(
+ "<=>",
+ Array(LiteralValue(1, IntegerType), LiteralValue(2, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = GreaterThan(Literal(1), Literal(2)),
+ v2Expr = new Predicate(
+ ">",
+ Array(LiteralValue(1, IntegerType), LiteralValue(2, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = GreaterThanOrEqual(Literal(1), Literal(2)),
+ v2Expr = new Predicate(
+ ">=",
+ Array(LiteralValue(1, IntegerType), LiteralValue(2, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = LessThan(Literal(1), Literal(2)),
+ v2Expr = new Predicate(
+ "<",
+ Array(LiteralValue(1, IntegerType), LiteralValue(2, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = LessThanOrEqual(Literal(1), Literal(2)),
+ v2Expr = new Predicate(
+ "<=",
+ Array(LiteralValue(1, IntegerType), LiteralValue(2, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Not(EqualTo(Literal(1), Literal(2))),
+ v2Expr = new Predicate(
+ "<>",
+ Array(LiteralValue(1, IntegerType), LiteralValue(2, IntegerType))))
+
+ checkRoundTripConversion(
+ catalystExpr = And(Literal.TrueLiteral, Literal.FalseLiteral),
+ v2Expr = new V2And(new AlwaysTrue, new AlwaysFalse))
+
+ checkRoundTripConversion(
+ catalystExpr = Or(Literal.TrueLiteral, Literal.FalseLiteral),
+ v2Expr = new V2Or(new AlwaysTrue, new AlwaysFalse))
+
+ checkRoundTripConversion(
+ catalystExpr = StartsWith($"a".string, Literal("foo")),
+ v2Expr = new Predicate(
+ "STARTS_WITH",
+ Array(FieldReference("a"), LiteralValue(UTF8String.fromString("foo"), StringType))))
+
+ checkRoundTripConversion(
+ catalystExpr = EndsWith($"a".string, Literal("bar")),
+ v2Expr = new Predicate(
+ "ENDS_WITH",
+ Array(FieldReference("a"), LiteralValue(UTF8String.fromString("bar"), StringType))))
+
+ checkRoundTripConversion(
+ catalystExpr = Contains($"a".string, Literal("baz")),
+ v2Expr = new Predicate(
+ "CONTAINS",
+ Array(FieldReference("a"), LiteralValue(UTF8String.fromString("baz"), StringType))))
+
+ checkRoundTripConversion(
+ catalystExpr = In($"a".int, Seq(Literal(1), Literal(2), Literal(3))),
+ v2Expr = new Predicate("IN", Array(
+ FieldReference("a"),
+ LiteralValue(1, IntegerType),
+ LiteralValue(2, IntegerType),
+ LiteralValue(3, IntegerType))))
+ }
+
/**
* Translate the given Catalyst [[Expression]] into data source V2 [[Predicate]]
* then verify against the given [[Predicate]].
@@ -326,4 +723,34 @@ class DataSourceV2StrategySuite extends PlanTest with SharedSparkSession {
DataSourceV2Strategy.translateFilterV2(catalystFilter)
}
}
+
+ private def checkV2Conversion(
+ catalystExpr: Expression,
+ v2Expr: V2Expression,
+ isPredicate: Boolean = false): Unit = {
+ val v2ExprActual = new V2ExpressionBuilder(catalystExpr, isPredicate).build().getOrElse {
+ fail(s"can't convert to V2 expression: $catalystExpr")
+ }
+ assert(v2ExprActual == v2Expr, "V2 expressions must match")
+ }
+
+ private def checkCatalystConversion(
+ v2Expr: V2Expression,
+ catalystExpr: Expression): Unit = {
+ val catalystExprActual = V2ExpressionUtils.toCatalyst(v2Expr).getOrElse {
+ fail(s"can't convert to Catalyst expression: $v2Expr")
+ }
+ val catalystExprExpected = catalystExpr.transform {
+ case attr: Attribute => UnresolvedAttribute(attr.name)
+ }
+ assert(catalystExprActual == catalystExprExpected, "V1 expressions must match")
+ }
+
+ private def checkRoundTripConversion(
+ catalystExpr: Expression,
+ v2Expr: V2Expression,
+ isPredicate: Boolean = false): Unit = {
+ checkV2Conversion(catalystExpr, v2Expr, isPredicate)
+ checkCatalystConversion(v2Expr, catalystExpr)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org