You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@auron.apache.org by sh...@apache.org on 2026/04/06 17:56:37 UTC
(auron) branch master updated: [AURON #2166] Wire existing native scalar functions in NativeConverters (#2162)
This is an automated email from the ASF dual-hosted git repository.
shreyesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git
The following commit(s) were added to refs/heads/master by this push:
new 429d5a14 [AURON #2166] Wire existing native scalar functions in NativeConverters (#2162)
429d5a14 is described below
commit 429d5a1405fa1001b3f2dc8e21c48455bfa52588
Author: Shreyesh <sh...@gmail.com>
AuthorDate: Mon Apr 6 10:56:33 2026 -0700
[AURON #2166] Wire existing native scalar functions in NativeConverters (#2162)
# Which issue does this PR close?
Closes #2166
# Rationale for this change
These functions already had protobuf enum values and native
Rust/DataFusion implementations but were missing the Scala converter
wiring in NativeConverters.scala.
# What changes are included in this PR?
- Support for ascii, bit_length, chr, translate, replace, date_trunc
# Are there any user-facing changes?
# How was this patch tested?
- Unit tested through AuronFunctionSuite
---------
Co-authored-by: Claude Opus 4.6 <no...@anthropic.com>
---
.../org/apache/auron/AuronFunctionSuite.scala | 58 ++++++++++++++++++++++
.../apache/spark/sql/auron/NativeConverters.scala | 11 ++++
2 files changed, 69 insertions(+)
diff --git a/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronFunctionSuite.scala b/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronFunctionSuite.scala
index df070881..2df0d6c6 100644
--- a/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronFunctionSuite.scala
+++ b/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronFunctionSuite.scala
@@ -729,6 +729,64 @@ class AuronFunctionSuite extends AuronQueryTest with BaseAuronSQLSuite {
}
}
+ test("ascii function") {
+ withTable("t1") {
+ sql("create table t1(c1 string) using parquet")
+ sql("insert into t1 values('A'), ('abc'), (''), (null)")
+ checkSparkAnswerAndOperator("select ascii(c1) from t1")
+ }
+ }
+
+ test("bit_length function") {
+ withTable("t1") {
+ sql("create table t1(c1 string) using parquet")
+ sql("insert into t1 values('hello'), (''), (null), ('caf\\u00e9')")
+ checkSparkAnswerAndOperator("select bit_length(c1) from t1")
+ }
+ }
+
+ test("chr function") {
+ withTable("t1") {
+ sql("create table t1(c1 bigint) using parquet")
+ sql("insert into t1 values(65), (97), (48), (null)")
+ checkSparkAnswerAndOperator("select chr(c1) from t1")
+ }
+ }
+
+ test("translate function") {
+ withTable("t1") {
+ sql("create table t1(c1 string) using parquet")
+ sql("insert into t1 values('AaBbCc'), ('hello'), (''), (null)")
+ checkSparkAnswerAndOperator("select translate(c1, 'ABC', 'xyz') from t1")
+ }
+ }
+
+ test("replace function") {
+ withTable("t1") {
+ sql("create table t1(c1 string) using parquet")
+ sql("insert into t1 values('hello world'), ('aaa'), (''), (null)")
+ checkSparkAnswerAndOperator("select replace(c1, 'world', 'spark') from t1")
+ checkSparkAnswerAndOperator("select replace(c1, 'a', '') from t1")
+ }
+ }
+
+ test("date_trunc function") {
+ withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
+ withTable("t1") {
+ sql("create table t1(c1 timestamp) using parquet")
+ sql("""insert into t1 values
+ | (timestamp'2024-03-15 14:30:45'),
+ | (timestamp'2024-12-31 23:59:59'),
+ | (null)
+ |""".stripMargin)
+ checkSparkAnswerAndOperator("select date_trunc('year', c1) from t1")
+ checkSparkAnswerAndOperator("select date_trunc('month', c1) from t1")
+ checkSparkAnswerAndOperator("select date_trunc('day', c1) from t1")
+ checkSparkAnswerAndOperator("select date_trunc('hour', c1) from t1")
+ }
+ }
+ }
+
test("months_between function") {
withSQLConf("spark.sql.session.timeZone" -> "UTC") {
withTable("t1") {
diff --git a/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala b/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala
index 7db4374b..c0d12d60 100644
--- a/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala
+++ b/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala
@@ -891,6 +891,17 @@ object NativeConverters extends Logging {
buildScalarFunction(pb.ScalarFunction.FindInSet, e.children, e.dataType)
case e: Abs if e.dataType.isInstanceOf[FloatType] || e.dataType.isInstanceOf[DoubleType] =>
buildScalarFunction(pb.ScalarFunction.Abs, e.children, e.dataType)
+ case e: Ascii => buildScalarFunction(pb.ScalarFunction.Ascii, e.children, e.dataType)
+ case e: BitLength =>
+ buildScalarFunction(pb.ScalarFunction.BitLength, e.children, e.dataType)
+ case e: Chr => buildScalarFunction(pb.ScalarFunction.Chr, e.children, e.dataType)
+ case e: StringTranslate =>
+ buildScalarFunction(pb.ScalarFunction.Translate, e.children, e.dataType)
+ case e: StringReplace =>
+ buildScalarFunction(pb.ScalarFunction.Replace, e.children, e.dataType)
+ case e: TruncTimestamp =>
+ buildScalarFunction(pb.ScalarFunction.DateTrunc, e.children, e.dataType)
+
case e: OctetLength =>
buildScalarFunction(pb.ScalarFunction.OctetLength, e.children, e.dataType)
case Length(arg) if arg.dataType == StringType =>