You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@auron.apache.org by sh...@apache.org on 2026/04/06 17:56:37 UTC

(auron) branch master updated: [AURON #2166] Wire existing native scalar functions in NativeConverters (#2162)

This is an automated email from the ASF dual-hosted git repository.

shreyesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git


The following commit(s) were added to refs/heads/master by this push:
     new 429d5a14 [AURON #2166] Wire existing native scalar functions in NativeConverters (#2162)
429d5a14 is described below

commit 429d5a1405fa1001b3f2dc8e21c48455bfa52588
Author: Shreyesh <sh...@gmail.com>
AuthorDate: Mon Apr 6 10:56:33 2026 -0700

    [AURON #2166] Wire existing native scalar functions in NativeConverters (#2162)
    
    # Which issue does this PR close?
    
    Closes #2166
    
    # Rationale for this change
    These functions already had protobuf enum values and native
    Rust/DataFusion implementations but were missing the Scala converter
    wiring in NativeConverters.scala.
    
    # What changes are included in this PR?
    - Support for ascii, bit_length, chr, translate, replace, date_trunc
    # Are there any user-facing changes?
    
    # How was this patch tested?
    - Unit tested through AuronFunctionSuite
    
    ---------
    
    Co-authored-by: Claude Opus 4.6 <no...@anthropic.com>
---
 .../org/apache/auron/AuronFunctionSuite.scala      | 58 ++++++++++++++++++++++
 .../apache/spark/sql/auron/NativeConverters.scala  | 11 ++++
 2 files changed, 69 insertions(+)

diff --git a/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronFunctionSuite.scala b/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronFunctionSuite.scala
index df070881..2df0d6c6 100644
--- a/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronFunctionSuite.scala
+++ b/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronFunctionSuite.scala
@@ -729,6 +729,64 @@ class AuronFunctionSuite extends AuronQueryTest with BaseAuronSQLSuite {
     }
   }
 
+  test("ascii function") {
+    withTable("t1") {
+      sql("create table t1(c1 string) using parquet")
+      sql("insert into t1 values('A'), ('abc'), (''), (null)")
+      checkSparkAnswerAndOperator("select ascii(c1) from t1")
+    }
+  }
+
+  test("bit_length function") {
+    withTable("t1") {
+      sql("create table t1(c1 string) using parquet")
+      sql("insert into t1 values('hello'), (''), (null), ('caf\\u00e9')")
+      checkSparkAnswerAndOperator("select bit_length(c1) from t1")
+    }
+  }
+
+  test("chr function") {
+    withTable("t1") {
+      sql("create table t1(c1 bigint) using parquet")
+      sql("insert into t1 values(65), (97), (48), (null)")
+      checkSparkAnswerAndOperator("select chr(c1) from t1")
+    }
+  }
+
+  test("translate function") {
+    withTable("t1") {
+      sql("create table t1(c1 string) using parquet")
+      sql("insert into t1 values('AaBbCc'), ('hello'), (''), (null)")
+      checkSparkAnswerAndOperator("select translate(c1, 'ABC', 'xyz') from t1")
+    }
+  }
+
+  test("replace function") {
+    withTable("t1") {
+      sql("create table t1(c1 string) using parquet")
+      sql("insert into t1 values('hello world'), ('aaa'), (''), (null)")
+      checkSparkAnswerAndOperator("select replace(c1, 'world', 'spark') from t1")
+      checkSparkAnswerAndOperator("select replace(c1, 'a', '') from t1")
+    }
+  }
+
+  test("date_trunc function") {
+    withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
+      withTable("t1") {
+        sql("create table t1(c1 timestamp) using parquet")
+        sql("""insert into t1 values
+            |  (timestamp'2024-03-15 14:30:45'),
+            |  (timestamp'2024-12-31 23:59:59'),
+            |  (null)
+            |""".stripMargin)
+        checkSparkAnswerAndOperator("select date_trunc('year', c1) from t1")
+        checkSparkAnswerAndOperator("select date_trunc('month', c1) from t1")
+        checkSparkAnswerAndOperator("select date_trunc('day', c1) from t1")
+        checkSparkAnswerAndOperator("select date_trunc('hour', c1) from t1")
+      }
+    }
+  }
+
   test("months_between function") {
     withSQLConf("spark.sql.session.timeZone" -> "UTC") {
       withTable("t1") {
diff --git a/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala b/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala
index 7db4374b..c0d12d60 100644
--- a/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala
+++ b/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala
@@ -891,6 +891,17 @@ object NativeConverters extends Logging {
         buildScalarFunction(pb.ScalarFunction.FindInSet, e.children, e.dataType)
       case e: Abs if e.dataType.isInstanceOf[FloatType] || e.dataType.isInstanceOf[DoubleType] =>
         buildScalarFunction(pb.ScalarFunction.Abs, e.children, e.dataType)
+      case e: Ascii => buildScalarFunction(pb.ScalarFunction.Ascii, e.children, e.dataType)
+      case e: BitLength =>
+        buildScalarFunction(pb.ScalarFunction.BitLength, e.children, e.dataType)
+      case e: Chr => buildScalarFunction(pb.ScalarFunction.Chr, e.children, e.dataType)
+      case e: StringTranslate =>
+        buildScalarFunction(pb.ScalarFunction.Translate, e.children, e.dataType)
+      case e: StringReplace =>
+        buildScalarFunction(pb.ScalarFunction.Replace, e.children, e.dataType)
+      case e: TruncTimestamp =>
+        buildScalarFunction(pb.ScalarFunction.DateTrunc, e.children, e.dataType)
+
       case e: OctetLength =>
         buildScalarFunction(pb.ScalarFunction.OctetLength, e.children, e.dataType)
       case Length(arg) if arg.dataType == StringType =>