You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by mb...@apache.org on 2026/05/29 19:54:08 UTC
(datafusion-comet) branch main updated: feat: route Upper/Lower/InitCap through codegen dispatcher (#4499)
This is an automated email from the ASF dual-hosted git repository.
mbutrovich pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 259686179 feat: route Upper/Lower/InitCap through codegen dispatcher (#4499)
259686179 is described below
commit 25968617969a73e47273755c466e2af5bcb96ab1
Author: Andy Grove <ag...@apache.org>
AuthorDate: Fri May 29 13:54:03 2026 -0600
feat: route Upper/Lower/InitCap through codegen dispatcher (#4499)
---
.../scala/org/apache/comet/serde/strings.scala | 45 +++---
.../sql-tests/expressions/string/init_cap.sql | 21 ++-
.../sql-tests/expressions/string/lower.sql | 18 ++-
.../sql-tests/expressions/string/upper.sql | 18 ++-
.../approved-plans-v1_4/q24a/extended.txt | 174 ++++++++++-----------
.../approved-plans-v1_4/q24b/extended.txt | 174 ++++++++++-----------
.../approved-plans-v2_7/q24/extended.txt | 173 ++++++++++----------
.../spark/sql/comet/CometPlanStabilitySuite.scala | 1 +
8 files changed, 332 insertions(+), 292 deletions(-)
diff --git a/spark/src/main/scala/org/apache/comet/serde/strings.scala b/spark/src/main/scala/org/apache/comet/serde/strings.scala
index 1d40d527a..e628e3742 100644
--- a/spark/src/main/scala/org/apache/comet/serde/strings.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/strings.scala
@@ -54,20 +54,19 @@ object CometStringRepeat extends CometExpressionSerde[StringRepeat] {
class CometCaseConversionBase[T <: Expression](function: String)
extends CometScalarFunction[T](function) {
- override def getIncompatibleReasons(): Seq[String] = Seq(
- "Results can vary depending on locale and character set." +
- s" Requires `${CometConf.COMET_CASE_CONVERSION_ENABLED.key}=true` to enable.")
+ override def getSupportLevel(expr: T): SupportLevel = Compatible()
override def convert(expr: T, inputs: Seq[Attribute], binding: Boolean): Option[Expr] = {
- if (!CometConf.COMET_CASE_CONVERSION_ENABLED.get()) {
- withInfo(
- expr,
- "Comet is not compatible with Spark for case conversion in " +
- s"locale-specific cases. Set ${CometConf.COMET_CASE_CONVERSION_ENABLED.key}=true " +
- "to enable it anyway.")
- return None
+ if (CometConf.COMET_CASE_CONVERSION_ENABLED.get()) {
+ // Native scalar function: faster but does not match Spark for locale-specific characters
+ // (e.g. Turkish dotted/dotless I). Opt-in.
+ super.convert(expr, inputs, binding)
+ } else {
+ // Default: route through the codegen dispatcher so Spark's own doGenCode runs inside the
+ // Comet pipeline. This guarantees Spark-compatible behavior across 3.4 / 3.5 / 4.0.
+ // Falls through to Spark when the dispatcher is disabled.
+ CometScalaUDF.emitJvmCodegenDispatch(expr, inputs, binding)
}
- super.convert(expr, inputs, binding)
}
}
@@ -86,20 +85,20 @@ object CometLength extends CometScalarFunction[Length]("length") {
object CometInitCap extends CometScalarFunction[InitCap]("initcap") {
- override def getIncompatibleReasons(): Seq[String] = Seq(
- "Treats hyphen as a word separator (e.g. `robert rose-smith` produces `Robert Rose-Smith`" +
- " instead of Spark's `Robert Rose-smith`)" +
- " (https://github.com/apache/datafusion-comet/issues/1052)")
-
- override def getSupportLevel(expr: InitCap): SupportLevel = {
- // Behavior differs from Spark. One example is that for the input "robert rose-smith", Spark
- // will produce "Robert Rose-smith", but Comet will produce "Robert Rose-Smith".
- // https://github.com/apache/datafusion-comet/issues/1052
- Incompatible(None)
- }
+ override def getSupportLevel(expr: InitCap): SupportLevel = Compatible()
override def convert(expr: InitCap, inputs: Seq[Attribute], binding: Boolean): Option[Expr] = {
- super.convert(expr, inputs, binding)
+ if (CometConf.isExprAllowIncompat(getExprConfigName(expr))) {
+ // Native path: faster but treats hyphen as a word separator (e.g.
+ // `robert rose-smith` produces `Robert Rose-Smith` instead of Spark's `Robert Rose-smith`).
+ // https://github.com/apache/datafusion-comet/issues/1052
+ super.convert(expr, inputs, binding)
+ } else {
+ // Default: route through the codegen dispatcher so Spark's own doGenCode runs inside the
+ // Comet pipeline. This guarantees Spark-compatible behavior across 3.4 / 3.5 / 4.0.
+ // Falls through to Spark when the dispatcher is disabled.
+ CometScalaUDF.emitJvmCodegenDispatch(expr, inputs, binding)
+ }
}
}
diff --git a/spark/src/test/resources/sql-tests/expressions/string/init_cap.sql b/spark/src/test/resources/sql-tests/expressions/string/init_cap.sql
index e0df1a61a..2c4e71634 100644
--- a/spark/src/test/resources/sql-tests/expressions/string/init_cap.sql
+++ b/spark/src/test/resources/sql-tests/expressions/string/init_cap.sql
@@ -15,11 +15,30 @@
-- specific language governing permissions and limitations
-- under the License.
+-- Routes InitCap through the codegen dispatcher so behavior matches Spark exactly,
+-- including the hyphen-as-word-separator case where the Rust scalar function diverges
+-- (see https://github.com/apache/datafusion-comet/issues/1052).
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
+
statement
CREATE TABLE test_initcap(s string) USING parquet
statement
INSERT INTO test_initcap VALUES ('hello world'), ('HELLO WORLD'), (''), (NULL), ('hello-world'), ('123abc'), (' spaces ')
-query expect_fallback(not fully compatible with Spark)
+query
SELECT initcap(s) FROM test_initcap
+
+-- literal arguments
+query
+SELECT initcap('hello world'), initcap(''), initcap(NULL)
+
+-- hyphen and other word separators - the divergence the codegen dispatcher fixes
+statement
+CREATE TABLE test_initcap_separators(s string) USING parquet
+
+statement
+INSERT INTO test_initcap_separators VALUES ('robert rose-smith'), ('foo.bar'), ('a_b_c'), ("o'reilly")
+
+query
+SELECT initcap(s) FROM test_initcap_separators
diff --git a/spark/src/test/resources/sql-tests/expressions/string/lower.sql b/spark/src/test/resources/sql-tests/expressions/string/lower.sql
index 723325182..e21bf7710 100644
--- a/spark/src/test/resources/sql-tests/expressions/string/lower.sql
+++ b/spark/src/test/resources/sql-tests/expressions/string/lower.sql
@@ -15,15 +15,29 @@
-- specific language governing permissions and limitations
-- under the License.
+-- Routes Lower through the codegen dispatcher so behavior matches Spark exactly,
+-- including locale-specific case mappings that the Rust scalar function does not implement.
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
+
statement
CREATE TABLE test_lower(s string) USING parquet
statement
INSERT INTO test_lower VALUES ('HELLO'), ('hello'), ('Hello World'), (''), (NULL), ('123ABC')
-query expect_fallback(case conversion)
+query
SELECT lower(s) FROM test_lower
-- literal arguments
-query expect_fallback(case conversion)
+query
SELECT lower('HELLO'), lower(''), lower(NULL)
+
+-- locale-sensitive characters: Greek sigma and Turkish dotted I
+statement
+CREATE TABLE test_lower_unicode(s string) USING parquet
+
+statement
+INSERT INTO test_lower_unicode VALUES ('ΣIGMA'), ('İSTANBUL'), ('GROSSE'), ('CAFÉ')
+
+query
+SELECT lower(s) FROM test_lower_unicode
diff --git a/spark/src/test/resources/sql-tests/expressions/string/upper.sql b/spark/src/test/resources/sql-tests/expressions/string/upper.sql
index 6a15efe59..c4f64376e 100644
--- a/spark/src/test/resources/sql-tests/expressions/string/upper.sql
+++ b/spark/src/test/resources/sql-tests/expressions/string/upper.sql
@@ -15,15 +15,29 @@
-- specific language governing permissions and limitations
-- under the License.
+-- Routes Upper through the codegen dispatcher so behavior matches Spark exactly,
+-- including locale-specific case mappings that the Rust scalar function does not implement.
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
+
statement
CREATE TABLE test_upper(s string) USING parquet
statement
INSERT INTO test_upper VALUES ('hello'), ('HELLO'), ('Hello World'), (''), (NULL), ('123abc')
-query expect_fallback(case conversion)
+query
SELECT upper(s) FROM test_upper
-- literal arguments
-query expect_fallback(case conversion)
+query
SELECT upper('hello'), upper(''), upper(NULL)
+
+-- locale-sensitive characters: German sharp s and Turkish dotted/dotless I
+statement
+CREATE TABLE test_upper_unicode(s string) USING parquet
+
+statement
+INSERT INTO test_upper_unicode VALUES ('straße'), ('istanbul'), ('İstanbul'), ('finish')
+
+query
+SELECT upper(s) FROM test_upper_unicode
diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/extended.txt
index 4fd8252b2..7dc6cd033 100644
--- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/extended.txt
+++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/extended.txt
@@ -1,92 +1,90 @@
-Filter
-: +- Subquery
-: +- HashAggregate [COMET: Spark Final aggregate without Comet Partial requires compatible intermediate buffer formats]
-: +- Exchange
-: +- HashAggregate
-: +- HashAggregate [COMET: Spark Final aggregate without Comet Partial requires compatible intermediate buffer formats]
-: +- Exchange
-: +- HashAggregate
-: +- Project
-: +- BroadcastHashJoin [COMET: Comet is not compatible with Spark for case conversion in locale-specific cases. Set spark.comet.caseConversion.enabled=true to enable it anyway.]
-: :- CometNativeColumnarToRow
-: : +- CometProject
-: : +- CometBroadcastHashJoin
-: : :- CometProject
-: : : +- CometBroadcastHashJoin
-: : : :- CometProject
-: : : : +- CometBroadcastHashJoin
-: : : : :- CometProject
-: : : : : +- CometSortMergeJoin
-: : : : : :- CometSort
-: : : : : : +- CometExchange
-: : : : : : +- CometProject
-: : : : : : +- CometFilter
-: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales
-: : : : : +- CometSort
-: : : : : +- CometExchange
-: : : : : +- CometProject
-: : : : : +- CometFilter
-: : : : : +- CometNativeScan parquet spark_catalog.default.store_returns
-: : : : +- CometBroadcastExchange
-: : : : +- CometProject
-: : : : +- CometFilter
-: : : : +- CometNativeScan parquet spark_catalog.default.store
-: : : +- CometBroadcastExchange
-: : : +- CometProject
-: : : +- CometFilter
-: : : +- CometNativeScan parquet spark_catalog.default.item
-: : +- CometBroadcastExchange
-: : +- CometProject
-: : +- CometFilter
-: : +- CometNativeScan parquet spark_catalog.default.customer
-: +- BroadcastExchange
-: +- CometNativeColumnarToRow
-: +- CometProject
-: +- CometFilter
-: +- CometNativeScan parquet spark_catalog.default.customer_address
-+- HashAggregate [COMET: Spark Final aggregate without Comet Partial requires compatible intermediate buffer formats]
- +- Exchange
- +- HashAggregate
- +- HashAggregate [COMET: Spark Final aggregate without Comet Partial requires compatible intermediate buffer formats]
- +- Exchange
- +- HashAggregate
- +- Project
- +- BroadcastHashJoin [COMET: Comet is not compatible with Spark for case conversion in locale-specific cases. Set spark.comet.caseConversion.enabled=true to enable it anyway.]
- :- CometNativeColumnarToRow
- : +- CometProject
- : +- CometBroadcastHashJoin
- : :- CometProject
- : : +- CometBroadcastHashJoin
- : : :- CometProject
- : : : +- CometBroadcastHashJoin
- : : : :- CometProject
- : : : : +- CometSortMergeJoin
- : : : : :- CometSort
- : : : : : +- CometExchange
- : : : : : +- CometProject
- : : : : : +- CometFilter
- : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales
- : : : : +- CometSort
- : : : : +- CometExchange
- : : : : +- CometProject
- : : : : +- CometFilter
- : : : : +- CometNativeScan parquet spark_catalog.default.store_returns
- : : : +- CometBroadcastExchange
- : : : +- CometProject
- : : : +- CometFilter
- : : : +- CometNativeScan parquet spark_catalog.default.store
- : : +- CometBroadcastExchange
- : : +- CometProject
- : : +- CometFilter
- : : +- CometNativeScan parquet spark_catalog.default.item
- : +- CometBroadcastExchange
- : +- CometProject
- : +- CometFilter
- : +- CometNativeScan parquet spark_catalog.default.customer
- +- BroadcastExchange
- +- CometNativeColumnarToRow
+CometNativeColumnarToRow
++- CometFilter
+ : +- Subquery
+ : +- CometNativeColumnarToRow
+ : +- CometHashAggregate
+ : +- CometExchange
+ : +- CometHashAggregate
+ : +- CometHashAggregate
+ : +- CometExchange
+ : +- CometHashAggregate
+ : +- CometProject
+ : +- CometBroadcastHashJoin
+ : :- CometProject
+ : : +- CometBroadcastHashJoin
+ : : :- CometProject
+ : : : +- CometBroadcastHashJoin
+ : : : :- CometProject
+ : : : : +- CometBroadcastHashJoin
+ : : : : :- CometProject
+ : : : : : +- CometSortMergeJoin
+ : : : : : :- CometSort
+ : : : : : : +- CometExchange
+ : : : : : : +- CometProject
+ : : : : : : +- CometFilter
+ : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales
+ : : : : : +- CometSort
+ : : : : : +- CometExchange
+ : : : : : +- CometProject
+ : : : : : +- CometFilter
+ : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns
+ : : : : +- CometBroadcastExchange
+ : : : : +- CometProject
+ : : : : +- CometFilter
+ : : : : +- CometNativeScan parquet spark_catalog.default.store
+ : : : +- CometBroadcastExchange
+ : : : +- CometProject
+ : : : +- CometFilter
+ : : : +- CometNativeScan parquet spark_catalog.default.item
+ : : +- CometBroadcastExchange
+ : : +- CometProject
+ : : +- CometFilter
+ : : +- CometNativeScan parquet spark_catalog.default.customer
+ : +- CometBroadcastExchange
+ : +- CometProject
+ : +- CometFilter
+ : +- CometNativeScan parquet spark_catalog.default.customer_address
+ +- CometHashAggregate
+ +- CometExchange
+ +- CometHashAggregate
+ +- CometHashAggregate
+ +- CometExchange
+ +- CometHashAggregate
+ +- CometProject
+ +- CometBroadcastHashJoin
+ :- CometProject
+ : +- CometBroadcastHashJoin
+ : :- CometProject
+ : : +- CometBroadcastHashJoin
+ : : :- CometProject
+ : : : +- CometBroadcastHashJoin
+ : : : :- CometProject
+ : : : : +- CometSortMergeJoin
+ : : : : :- CometSort
+ : : : : : +- CometExchange
+ : : : : : +- CometProject
+ : : : : : +- CometFilter
+ : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales
+ : : : : +- CometSort
+ : : : : +- CometExchange
+ : : : : +- CometProject
+ : : : : +- CometFilter
+ : : : : +- CometNativeScan parquet spark_catalog.default.store_returns
+ : : : +- CometBroadcastExchange
+ : : : +- CometProject
+ : : : +- CometFilter
+ : : : +- CometNativeScan parquet spark_catalog.default.store
+ : : +- CometBroadcastExchange
+ : : +- CometProject
+ : : +- CometFilter
+ : : +- CometNativeScan parquet spark_catalog.default.item
+ : +- CometBroadcastExchange
+ : +- CometProject
+ : +- CometFilter
+ : +- CometNativeScan parquet spark_catalog.default.customer
+ +- CometBroadcastExchange
+- CometProject
+- CometFilter
+- CometNativeScan parquet spark_catalog.default.customer_address
-Comet accelerated 66 out of 86 eligible operators (76%). Final plan contains 4 transitions between Spark and Comet.
\ No newline at end of file
+Comet accelerated 85 out of 86 eligible operators (98%). Final plan contains 2 transitions between Spark and Comet.
\ No newline at end of file
diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/extended.txt
index 4fd8252b2..7dc6cd033 100644
--- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/extended.txt
+++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/extended.txt
@@ -1,92 +1,90 @@
-Filter
-: +- Subquery
-: +- HashAggregate [COMET: Spark Final aggregate without Comet Partial requires compatible intermediate buffer formats]
-: +- Exchange
-: +- HashAggregate
-: +- HashAggregate [COMET: Spark Final aggregate without Comet Partial requires compatible intermediate buffer formats]
-: +- Exchange
-: +- HashAggregate
-: +- Project
-: +- BroadcastHashJoin [COMET: Comet is not compatible with Spark for case conversion in locale-specific cases. Set spark.comet.caseConversion.enabled=true to enable it anyway.]
-: :- CometNativeColumnarToRow
-: : +- CometProject
-: : +- CometBroadcastHashJoin
-: : :- CometProject
-: : : +- CometBroadcastHashJoin
-: : : :- CometProject
-: : : : +- CometBroadcastHashJoin
-: : : : :- CometProject
-: : : : : +- CometSortMergeJoin
-: : : : : :- CometSort
-: : : : : : +- CometExchange
-: : : : : : +- CometProject
-: : : : : : +- CometFilter
-: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales
-: : : : : +- CometSort
-: : : : : +- CometExchange
-: : : : : +- CometProject
-: : : : : +- CometFilter
-: : : : : +- CometNativeScan parquet spark_catalog.default.store_returns
-: : : : +- CometBroadcastExchange
-: : : : +- CometProject
-: : : : +- CometFilter
-: : : : +- CometNativeScan parquet spark_catalog.default.store
-: : : +- CometBroadcastExchange
-: : : +- CometProject
-: : : +- CometFilter
-: : : +- CometNativeScan parquet spark_catalog.default.item
-: : +- CometBroadcastExchange
-: : +- CometProject
-: : +- CometFilter
-: : +- CometNativeScan parquet spark_catalog.default.customer
-: +- BroadcastExchange
-: +- CometNativeColumnarToRow
-: +- CometProject
-: +- CometFilter
-: +- CometNativeScan parquet spark_catalog.default.customer_address
-+- HashAggregate [COMET: Spark Final aggregate without Comet Partial requires compatible intermediate buffer formats]
- +- Exchange
- +- HashAggregate
- +- HashAggregate [COMET: Spark Final aggregate without Comet Partial requires compatible intermediate buffer formats]
- +- Exchange
- +- HashAggregate
- +- Project
- +- BroadcastHashJoin [COMET: Comet is not compatible with Spark for case conversion in locale-specific cases. Set spark.comet.caseConversion.enabled=true to enable it anyway.]
- :- CometNativeColumnarToRow
- : +- CometProject
- : +- CometBroadcastHashJoin
- : :- CometProject
- : : +- CometBroadcastHashJoin
- : : :- CometProject
- : : : +- CometBroadcastHashJoin
- : : : :- CometProject
- : : : : +- CometSortMergeJoin
- : : : : :- CometSort
- : : : : : +- CometExchange
- : : : : : +- CometProject
- : : : : : +- CometFilter
- : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales
- : : : : +- CometSort
- : : : : +- CometExchange
- : : : : +- CometProject
- : : : : +- CometFilter
- : : : : +- CometNativeScan parquet spark_catalog.default.store_returns
- : : : +- CometBroadcastExchange
- : : : +- CometProject
- : : : +- CometFilter
- : : : +- CometNativeScan parquet spark_catalog.default.store
- : : +- CometBroadcastExchange
- : : +- CometProject
- : : +- CometFilter
- : : +- CometNativeScan parquet spark_catalog.default.item
- : +- CometBroadcastExchange
- : +- CometProject
- : +- CometFilter
- : +- CometNativeScan parquet spark_catalog.default.customer
- +- BroadcastExchange
- +- CometNativeColumnarToRow
+CometNativeColumnarToRow
++- CometFilter
+ : +- Subquery
+ : +- CometNativeColumnarToRow
+ : +- CometHashAggregate
+ : +- CometExchange
+ : +- CometHashAggregate
+ : +- CometHashAggregate
+ : +- CometExchange
+ : +- CometHashAggregate
+ : +- CometProject
+ : +- CometBroadcastHashJoin
+ : :- CometProject
+ : : +- CometBroadcastHashJoin
+ : : :- CometProject
+ : : : +- CometBroadcastHashJoin
+ : : : :- CometProject
+ : : : : +- CometBroadcastHashJoin
+ : : : : :- CometProject
+ : : : : : +- CometSortMergeJoin
+ : : : : : :- CometSort
+ : : : : : : +- CometExchange
+ : : : : : : +- CometProject
+ : : : : : : +- CometFilter
+ : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales
+ : : : : : +- CometSort
+ : : : : : +- CometExchange
+ : : : : : +- CometProject
+ : : : : : +- CometFilter
+ : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns
+ : : : : +- CometBroadcastExchange
+ : : : : +- CometProject
+ : : : : +- CometFilter
+ : : : : +- CometNativeScan parquet spark_catalog.default.store
+ : : : +- CometBroadcastExchange
+ : : : +- CometProject
+ : : : +- CometFilter
+ : : : +- CometNativeScan parquet spark_catalog.default.item
+ : : +- CometBroadcastExchange
+ : : +- CometProject
+ : : +- CometFilter
+ : : +- CometNativeScan parquet spark_catalog.default.customer
+ : +- CometBroadcastExchange
+ : +- CometProject
+ : +- CometFilter
+ : +- CometNativeScan parquet spark_catalog.default.customer_address
+ +- CometHashAggregate
+ +- CometExchange
+ +- CometHashAggregate
+ +- CometHashAggregate
+ +- CometExchange
+ +- CometHashAggregate
+ +- CometProject
+ +- CometBroadcastHashJoin
+ :- CometProject
+ : +- CometBroadcastHashJoin
+ : :- CometProject
+ : : +- CometBroadcastHashJoin
+ : : :- CometProject
+ : : : +- CometBroadcastHashJoin
+ : : : :- CometProject
+ : : : : +- CometSortMergeJoin
+ : : : : :- CometSort
+ : : : : : +- CometExchange
+ : : : : : +- CometProject
+ : : : : : +- CometFilter
+ : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales
+ : : : : +- CometSort
+ : : : : +- CometExchange
+ : : : : +- CometProject
+ : : : : +- CometFilter
+ : : : : +- CometNativeScan parquet spark_catalog.default.store_returns
+ : : : +- CometBroadcastExchange
+ : : : +- CometProject
+ : : : +- CometFilter
+ : : : +- CometNativeScan parquet spark_catalog.default.store
+ : : +- CometBroadcastExchange
+ : : +- CometProject
+ : : +- CometFilter
+ : : +- CometNativeScan parquet spark_catalog.default.item
+ : +- CometBroadcastExchange
+ : +- CometProject
+ : +- CometFilter
+ : +- CometNativeScan parquet spark_catalog.default.customer
+ +- CometBroadcastExchange
+- CometProject
+- CometFilter
+- CometNativeScan parquet spark_catalog.default.customer_address
-Comet accelerated 66 out of 86 eligible operators (76%). Final plan contains 4 transitions between Spark and Comet.
\ No newline at end of file
+Comet accelerated 85 out of 86 eligible operators (98%). Final plan contains 2 transitions between Spark and Comet.
\ No newline at end of file
diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/extended.txt
index a94d9e453..7418929ba 100644
--- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/extended.txt
+++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/extended.txt
@@ -1,95 +1,92 @@
CometNativeColumnarToRow
+- CometSort
- +- CometColumnarExchange
- +- Filter
+ +- CometExchange
+ +- CometFilter
: +- Subquery
- : +- HashAggregate [COMET: Spark Final aggregate without Comet Partial requires compatible intermediate buffer formats]
- : +- Exchange
- : +- HashAggregate
- : +- HashAggregate [COMET: Spark Final aggregate without Comet Partial requires compatible intermediate buffer formats]
- : +- Exchange
- : +- HashAggregate
- : +- Project
- : +- BroadcastHashJoin [COMET: Comet is not compatible with Spark for case conversion in locale-specific cases. Set spark.comet.caseConversion.enabled=true to enable it anyway.]
- : :- CometNativeColumnarToRow
- : : +- CometProject
- : : +- CometBroadcastHashJoin
- : : :- CometProject
- : : : +- CometBroadcastHashJoin
- : : : :- CometProject
- : : : : +- CometBroadcastHashJoin
- : : : : :- CometProject
- : : : : : +- CometSortMergeJoin
- : : : : : :- CometSort
- : : : : : : +- CometExchange
- : : : : : : +- CometProject
- : : : : : : +- CometFilter
- : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales
- : : : : : +- CometSort
- : : : : : +- CometExchange
- : : : : : +- CometProject
- : : : : : +- CometFilter
- : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns
- : : : : +- CometBroadcastExchange
- : : : : +- CometProject
- : : : : +- CometFilter
- : : : : +- CometNativeScan parquet spark_catalog.default.store
- : : : +- CometBroadcastExchange
- : : : +- CometProject
- : : : +- CometFilter
- : : : +- CometNativeScan parquet spark_catalog.default.item
- : : +- CometBroadcastExchange
- : : +- CometProject
- : : +- CometFilter
- : : +- CometNativeScan parquet spark_catalog.default.customer
- : +- BroadcastExchange
- : +- CometNativeColumnarToRow
+ : +- CometNativeColumnarToRow
+ : +- CometHashAggregate
+ : +- CometExchange
+ : +- CometHashAggregate
+ : +- CometHashAggregate
+ : +- CometExchange
+ : +- CometHashAggregate
+ : +- CometProject
+ : +- CometBroadcastHashJoin
+ : :- CometProject
+ : : +- CometBroadcastHashJoin
+ : : :- CometProject
+ : : : +- CometBroadcastHashJoin
+ : : : :- CometProject
+ : : : : +- CometBroadcastHashJoin
+ : : : : :- CometProject
+ : : : : : +- CometSortMergeJoin
+ : : : : : :- CometSort
+ : : : : : : +- CometExchange
+ : : : : : : +- CometProject
+ : : : : : : +- CometFilter
+ : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales
+ : : : : : +- CometSort
+ : : : : : +- CometExchange
+ : : : : : +- CometProject
+ : : : : : +- CometFilter
+ : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns
+ : : : : +- CometBroadcastExchange
+ : : : : +- CometProject
+ : : : : +- CometFilter
+ : : : : +- CometNativeScan parquet spark_catalog.default.store
+ : : : +- CometBroadcastExchange
+ : : : +- CometProject
+ : : : +- CometFilter
+ : : : +- CometNativeScan parquet spark_catalog.default.item
+ : : +- CometBroadcastExchange
+ : : +- CometProject
+ : : +- CometFilter
+ : : +- CometNativeScan parquet spark_catalog.default.customer
+ : +- CometBroadcastExchange
: +- CometProject
: +- CometFilter
: +- CometNativeScan parquet spark_catalog.default.customer_address
- +- HashAggregate [COMET: Spark Final aggregate without Comet Partial requires compatible intermediate buffer formats]
- +- Exchange
- +- HashAggregate
- +- HashAggregate [COMET: Spark Final aggregate without Comet Partial requires compatible intermediate buffer formats]
- +- Exchange
- +- HashAggregate
- +- Project
- +- BroadcastHashJoin [COMET: Comet is not compatible with Spark for case conversion in locale-specific cases. Set spark.comet.caseConversion.enabled=true to enable it anyway.]
- :- CometNativeColumnarToRow
- : +- CometProject
- : +- CometBroadcastHashJoin
- : :- CometProject
- : : +- CometBroadcastHashJoin
- : : :- CometProject
- : : : +- CometBroadcastHashJoin
- : : : :- CometProject
- : : : : +- CometSortMergeJoin
- : : : : :- CometSort
- : : : : : +- CometExchange
- : : : : : +- CometProject
- : : : : : +- CometFilter
- : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales
- : : : : +- CometSort
- : : : : +- CometExchange
- : : : : +- CometProject
- : : : : +- CometFilter
- : : : : +- CometNativeScan parquet spark_catalog.default.store_returns
- : : : +- CometBroadcastExchange
- : : : +- CometProject
- : : : +- CometFilter
- : : : +- CometNativeScan parquet spark_catalog.default.store
- : : +- CometBroadcastExchange
- : : +- CometProject
- : : +- CometFilter
- : : +- CometNativeScan parquet spark_catalog.default.item
- : +- CometBroadcastExchange
- : +- CometProject
- : +- CometFilter
- : +- CometNativeScan parquet spark_catalog.default.customer
- +- BroadcastExchange
- +- CometNativeColumnarToRow
- +- CometProject
- +- CometFilter
- +- CometNativeScan parquet spark_catalog.default.customer_address
+ +- CometHashAggregate
+ +- CometExchange
+ +- CometHashAggregate
+ +- CometHashAggregate
+ +- CometExchange
+ +- CometHashAggregate
+ +- CometProject
+ +- CometBroadcastHashJoin
+ :- CometProject
+ : +- CometBroadcastHashJoin
+ : :- CometProject
+ : : +- CometBroadcastHashJoin
+ : : :- CometProject
+ : : : +- CometBroadcastHashJoin
+ : : : :- CometProject
+ : : : : +- CometSortMergeJoin
+ : : : : :- CometSort
+ : : : : : +- CometExchange
+ : : : : : +- CometProject
+ : : : : : +- CometFilter
+ : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales
+ : : : : +- CometSort
+ : : : : +- CometExchange
+ : : : : +- CometProject
+ : : : : +- CometFilter
+ : : : : +- CometNativeScan parquet spark_catalog.default.store_returns
+ : : : +- CometBroadcastExchange
+ : : : +- CometProject
+ : : : +- CometFilter
+ : : : +- CometNativeScan parquet spark_catalog.default.store
+ : : +- CometBroadcastExchange
+ : : +- CometProject
+ : : +- CometFilter
+ : : +- CometNativeScan parquet spark_catalog.default.item
+ : +- CometBroadcastExchange
+ : +- CometProject
+ : +- CometFilter
+ : +- CometNativeScan parquet spark_catalog.default.customer
+ +- CometBroadcastExchange
+ +- CometProject
+ +- CometFilter
+ +- CometNativeScan parquet spark_catalog.default.customer_address
-Comet accelerated 68 out of 88 eligible operators (77%). Final plan contains 5 transitions between Spark and Comet.
\ No newline at end of file
+Comet accelerated 87 out of 88 eligible operators (98%). Final plan contains 2 transitions between Spark and Comet.
\ No newline at end of file
diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala
index c75669515..117ce67b2 100644
--- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala
@@ -318,6 +318,7 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa
conf.set(CometConf.COMET_EXEC_ENABLED.key, "true")
conf.set(CometConf.COMET_ONHEAP_MEMORY_OVERHEAD.key, "1g")
conf.set(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, "true")
+ conf.set(CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key, "true")
new TestSparkSession(new SparkContext("local[1]", this.getClass.getCanonicalName, conf))
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org