You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ch...@apache.org on 2026/04/15 11:17:28 UTC
(gluten) 05/29: ci(ansi): simplify CastWithAnsiOffSuite + add Spark 4.0 ANSI CI jobs
This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch feature/ansi-ci-baseline
in repository https://gitbox.apache.org/repos/asf/gluten.git
commit 9113b152e9b93e7ae7f876de180d379d54932333
Author: Chang chen <ch...@microsoft.com>
AuthorDate: Mon Mar 30 10:26:11 2026 +0000
ci(ansi): simplify CastWithAnsiOffSuite + add Spark 4.0 ANSI CI jobs
- Remove all redundant overrides from GlutenCastWithAnsiOffSuite (cast(),
testGluten rewrites, UDT registration) for both spark40 and spark41,
keeping only beforeAll + GlutenTestsTrait mixin
- Remove corresponding excludes from VeloxTestSettings for both versions
- Enable GlutenCastWithAnsiOnSuite for spark40
- Add --jvm-arg support to run-scala-test.sh
- Add Spark 4.0 backends-velox and spark-ut ANSI CI jobs to workflow
Co-Authored-By: Claude Opus 4.6 <no...@anthropic.com>
---
.github/workflows/velox_backend_ansi.yml | 155 ++++++++++++
dev/run-scala-test.sh | 13 +
.../gluten/utils/velox/VeloxTestSettings.scala | 12 +-
.../expressions/GlutenCastWithAnsiOffSuite.scala | 267 --------------------
.../gluten/utils/velox/VeloxTestSettings.scala | 10 -
.../expressions/GlutenCastWithAnsiOffSuite.scala | 274 +--------------------
6 files changed, 170 insertions(+), 561 deletions(-)
diff --git a/.github/workflows/velox_backend_ansi.yml b/.github/workflows/velox_backend_ansi.yml
index 51af6edb31..f5e3679064 100644
--- a/.github/workflows/velox_backend_ansi.yml
+++ b/.github/workflows/velox_backend_ansi.yml
@@ -242,3 +242,158 @@ jobs:
**/target/*.log
**/gluten-ut/**/hs_err_*.log
**/gluten-ut/**/core.*
+
+ spark-test-backends-velox-ansi-spark40:
+ needs: build-native-lib
+ runs-on: ubuntu-22.04
+ env:
+ SPARK_TESTING: true
+ container: apache/gluten:centos-9-jdk17
+ steps:
+ - uses: actions/checkout@v4
+ - name: Download Native Lib
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-native-lib-ansi-${{github.sha}}
+ path: ./cpp/build/
+ - name: Download Arrow Jars
+ uses: actions/download-artifact@v4
+ with:
+ name: arrow-jars-ansi-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
+ - name: Prepare
+ run: |
+ dnf install -y python3.11 python3.11-pip python3.11-devel && \
+ alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
+ alternatives --set python3 /usr/bin/python3.11 && \
+ pip3 install setuptools==77.0.3 && \
+ pip3 install pyspark==3.5.5 cython && \
+ pip3 install pandas==2.2.3 pyarrow==20.0.0
+ - name: Prepare Spark Resources
+ run: |
+ rm -rf /opt/shims/spark40
+ bash .github/workflows/util/install-resources.sh 4.0
+ mv /opt/shims/spark40/spark_home/assembly/target/scala-2.12 /opt/shims/spark40/spark_home/assembly/target/scala-2.13
+ - name: "Spark 4.0 backends-velox Tests (ANSI ON)"
+ run: |
+ set -o pipefail
+ cd $GITHUB_WORKSPACE/
+ export SPARK_SCALA_VERSION=2.13
+ yum install -y java-17-openjdk-devel
+ export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+ export PATH=$JAVA_HOME/bin:$PATH
+ echo "SPARK_ANSI_SQL_MODE=$SPARK_ANSI_SQL_MODE"
+ java -version
+ $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox \
+ -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/ -Dspark.gluten.sql.ansiFallback.enabled=false" \
+ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \
+ 2>&1 | tee backends-velox-spark40-test-output.log
+ - name: "Parse test results"
+ if: always()
+ run: |
+ echo "========================================="
+ echo " backends-velox (Spark 4.0, ANSI=$SPARK_ANSI_SQL_MODE)"
+ echo "========================================="
+ echo ""
+ echo "--- Test Summary ---"
+ grep -E "Tests run:.*Failures:|BUILD " backends-velox-spark40-test-output.log || echo "(no summary found)"
+ echo ""
+ echo "--- Failed Tests ---"
+ grep -B1 "<<< FAIL!" backends-velox-spark40-test-output.log || echo "(no failures)"
+ echo ""
+ echo "--- Error Tests ---"
+ grep -B1 "<<< ERROR!" backends-velox-spark40-test-output.log || echo "(no errors)"
+ - name: Upload test report
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: spark-test-backends-velox-ansi-spark40-report
+ path: |
+ **/surefire-reports/TEST-*.xml
+ backends-velox-spark40-test-output.log
+ - name: Upload log files
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: spark-test-backends-velox-ansi-spark40-logs
+ path: |
+ **/target/*.log
+
+ spark-test-spark-ut-ansi-spark40:
+ needs: build-native-lib
+ runs-on: ubuntu-22.04
+ env:
+ SPARK_TESTING: true
+ container: apache/gluten:centos-9-jdk17
+ steps:
+ - uses: actions/checkout@v4
+ - name: Download Native Lib
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-native-lib-ansi-${{github.sha}}
+ path: ./cpp/build/
+ - name: Download Arrow Jars
+ uses: actions/download-artifact@v4
+ with:
+ name: arrow-jars-ansi-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
+ - name: Prepare
+ run: |
+ dnf install -y python3.11 python3.11-pip python3.11-devel && \
+ alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
+ alternatives --set python3 /usr/bin/python3.11 && \
+ pip3 install setuptools==77.0.3 && \
+ pip3 install pyspark==3.5.5 cython && \
+ pip3 install pandas==2.2.3 pyarrow==20.0.0
+ - name: Prepare Spark Resources
+ run: |
+ rm -rf /opt/shims/spark40
+ bash .github/workflows/util/install-resources.sh 4.0
+ mv /opt/shims/spark40/spark_home/assembly/target/scala-2.12 /opt/shims/spark40/spark_home/assembly/target/scala-2.13
+ - name: "Spark 4.0 spark-ut Tests (ANSI ON)"
+ run: |
+ set -o pipefail
+ cd $GITHUB_WORKSPACE/
+ export SPARK_SCALA_VERSION=2.13
+ yum install -y java-17-openjdk-devel
+ export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+ export PATH=$JAVA_HOME/bin:$PATH
+ echo "SPARK_ANSI_SQL_MODE=$SPARK_ANSI_SQL_MODE"
+ java -version
+ $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \
+ -DwildcardSuites='org.apache.spark.' -Dtest=none -DfailIfNoTests=false \
+ -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/ -Dspark.gluten.sql.ansiFallback.enabled=false" \
+ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \
+ 2>&1 | tee spark-ut-spark40-test-output.log
+ - name: "Parse test results"
+ if: always()
+ run: |
+ echo "========================================="
+ echo " spark-ut (Spark 4.0, ANSI=$SPARK_ANSI_SQL_MODE)"
+ echo "========================================="
+ echo ""
+ echo "--- Test Summary ---"
+ grep -E "Tests run:.*Failures:|BUILD " spark-ut-spark40-test-output.log || echo "(no summary found)"
+ echo ""
+ echo "--- Failed Tests ---"
+ grep -B1 "<<< FAIL!" spark-ut-spark40-test-output.log || echo "(no failures)"
+ echo ""
+ echo "--- Error Tests ---"
+ grep -B1 "<<< ERROR!" spark-ut-spark40-test-output.log || echo "(no errors)"
+ - name: Upload test report
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: spark-test-spark-ut-ansi-spark40-report
+ path: |
+ **/surefire-reports/TEST-*.xml
+ spark-ut-spark40-test-output.log
+ - name: Upload log files
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: spark-test-spark-ut-ansi-spark40-logs
+ path: |
+ **/target/*.log
+ **/gluten-ut/**/hs_err_*.log
+ **/gluten-ut/**/core.*
diff --git a/dev/run-scala-test.sh b/dev/run-scala-test.sh
index 2142c17e08..06aee7b25b 100755
--- a/dev/run-scala-test.sh
+++ b/dev/run-scala-test.sh
@@ -215,6 +215,7 @@ Optional:
--force Force Maven rebuild, bypass build cache
--profile Enable Maven profiler (reports in .profiler/)
--export-only Export classpath and exit (no test execution)
+ --jvm-arg <arg> Pass extra JVM argument to test process (repeatable)
--help Show this help message
Examples:
@@ -234,6 +235,12 @@ Examples:
-pl gluten-ut/spark40 \\
-s org.apache.spark.sql.GlutenTPCDSV1_4_PlanStabilitySuite \\
-s org.apache.spark.sql.GlutenTPCHPlanStabilitySuite
+
+ # Run with ANSI mode (disable fallback via --jvm-arg)
+ $0 --jvm-arg "-Dspark.gluten.sql.ansiFallback.enabled=false" \\
+ -Pjava-17,spark-4.1,scala-2.13,backends-velox,hadoop-3.3,spark-ut \\
+ -pl gluten-ut/spark41 \\
+ -s org.apache.spark.sql.catalyst.expressions.GlutenCastWithAnsiOffSuite
EOF
}
@@ -348,6 +355,7 @@ EXPORT_ONLY=false
ENABLE_CLEAN=false
FORCE_BUILD=false
USE_MVND=false
+EXTRA_JVM_ARGS=()
while [[ $# -gt 0 ]]; do
case $1 in
@@ -387,6 +395,10 @@ while [[ $# -gt 0 ]]; do
EXPORT_ONLY=true
shift
;;
+ --jvm-arg)
+ EXTRA_JVM_ARGS+=("$2")
+ shift 2
+ ;;
--help)
print_usage
exit 0
@@ -684,6 +696,7 @@ SPARK_TEST_HOME_ARG=""
JAVA_ARGS=(
${JVM_ARGS}
+ "${EXTRA_JVM_ARGS[@]}"
"-Dlog4j.configurationFile=file:${GLUTEN_HOME}/${MODULE}/src/test/resources/log4j2.properties"
${SPARK_TEST_HOME_ARG}
-cp "${PATHING_JAR}"
diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 498483cb65..8def0d92e8 100644
--- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -102,16 +102,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("SPARK-45786: Decimal multiply, divide, remainder, quot")
enableSuite[GlutenBitwiseExpressionsSuite]
enableSuite[GlutenCastWithAnsiOffSuite]
- .exclude(
- "Process Infinity, -Infinity, NaN in case insensitive manner" // +inf not supported in folly.
- )
- // Set timezone through config.
- .exclude("data type casting")
- // Revised by setting timezone through config and commented unsupported cases.
- .exclude("cast string to timestamp")
- .exclude("cast from timestamp II")
- .exclude("SPARK-36286: invalid string cast to timestamp")
- .exclude("SPARK-39749: cast Decimal to string")
enableSuite[GlutenTryCastSuite]
.exclude(
"Process Infinity, -Infinity, NaN in case insensitive manner" // +inf not supported in folly.
@@ -217,7 +207,7 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenBitmapExpressionUtilsSuite]
enableSuite[GlutenCallMethodViaReflectionSuite]
enableSuite[GlutenCanonicalizeSuite]
- // TODO: 4.x enableSuite[GlutenCastWithAnsiOnSuite] // 4 failures
+ enableSuite[GlutenCastWithAnsiOnSuite]
enableSuite[GlutenCodeGenerationSuite]
enableSuite[GlutenCodeGeneratorWithInterpretedFallbackSuite]
enableSuite[GlutenCollationExpressionSuite]
diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala
index 74c1b25ca2..291b986504 100644
--- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala
+++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala
@@ -17,14 +17,7 @@
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.GlutenTestsTrait
-import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, ALL_TIMEZONES, UTC, UTC_OPT}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp, millisToMicros, TimeZoneUTC}
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types._
-import org.apache.spark.util.DebuggableThreadUtils
-
-import java.sql.{Date, Timestamp}
-import java.util.{Calendar, TimeZone}
class GlutenCastWithAnsiOffSuite extends CastWithAnsiOffSuite with GlutenTestsTrait {
override def beforeAll(): Unit = {
@@ -35,264 +28,4 @@ class GlutenCastWithAnsiOffSuite extends CastWithAnsiOffSuite with GlutenTestsTr
// In Vanilla spark, the checkEvaluation method doesn't invoke RowEncoder.
conf.setConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO, true)
}
-
- override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] = None): Cast = {
- v match {
- case lit: Expression =>
- logDebug(s"Cast from: ${lit.dataType.typeName}, to: ${targetType.typeName}")
- Cast(lit, targetType, timeZoneId)
- case _ =>
- val lit = Literal(v)
- logDebug(s"Cast from: ${lit.dataType.typeName}, to: ${targetType.typeName}")
- Cast(lit, targetType, timeZoneId)
- }
- }
-
- // Register UDT For test("SPARK-32828")
- UDTRegistration.register(classOf[IExampleBaseType].getName, classOf[ExampleBaseTypeUDT].getName)
- UDTRegistration.register(classOf[IExampleSubType].getName, classOf[ExampleSubTypeUDT].getName)
-
- testGluten("missing cases - from boolean") {
- (DataTypeTestUtils.numericTypeWithoutDecimal ++ Set(BooleanType)).foreach {
- t =>
- t match {
- case BooleanType =>
- checkEvaluation(cast(cast(true, BooleanType), t), true)
- checkEvaluation(cast(cast(false, BooleanType), t), false)
- case _ =>
- checkEvaluation(cast(cast(true, BooleanType), t), 1)
- checkEvaluation(cast(cast(false, BooleanType), t), 0)
- }
- }
- }
-
- testGluten("missing cases - from byte") {
- DataTypeTestUtils.numericTypeWithoutDecimal.foreach {
- t =>
- checkEvaluation(cast(cast(0, ByteType), t), 0)
- checkEvaluation(cast(cast(-1, ByteType), t), -1)
- checkEvaluation(cast(cast(1, ByteType), t), 1)
- }
- }
-
- testGluten("missing cases - from short") {
- DataTypeTestUtils.numericTypeWithoutDecimal.foreach {
- t =>
- checkEvaluation(cast(cast(0, ShortType), t), 0)
- checkEvaluation(cast(cast(-1, ShortType), t), -1)
- checkEvaluation(cast(cast(1, ShortType), t), 1)
- }
- }
-
- testGluten("missing cases - date self check") {
- val d = Date.valueOf("1970-01-01")
- checkEvaluation(cast(d, DateType), d)
- }
-
- testGluten("data type casting") {
- val sd = "1970-01-01"
- val d = Date.valueOf(sd)
- val zts = sd + " 00:00:00"
- val sts = sd + " 00:00:02"
- val nts = sts + ".1"
- val ts = withDefaultTimeZone(UTC)(Timestamp.valueOf(nts))
-
- // SystemV timezones are a legacy way of specifying timezones in Unix-like OS.
- // It is not supported by Velox.
- for (tz <- ALL_TIMEZONES.filterNot(_.getId.contains("SystemV"))) {
- withSQLConf(
- SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz.getId
- ) {
- val timeZoneId = Option(tz.getId)
- var c = Calendar.getInstance(TimeZoneUTC)
- c.set(2015, 2, 8, 2, 30, 0)
- checkEvaluation(
- cast(
- cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
- TimestampType,
- timeZoneId),
- millisToMicros(c.getTimeInMillis))
- c = Calendar.getInstance(TimeZoneUTC)
- c.set(2015, 10, 1, 2, 30, 0)
- checkEvaluation(
- cast(
- cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
- TimestampType,
- timeZoneId),
- millisToMicros(c.getTimeInMillis))
- }
- }
-
- checkEvaluation(cast("abdef", StringType), "abdef")
- checkEvaluation(cast("12.65", DecimalType.SYSTEM_DEFAULT), Decimal(12.65))
-
- checkEvaluation(cast(cast(sd, DateType), StringType), sd)
- checkEvaluation(cast(cast(d, StringType), DateType), 0)
-
- withSQLConf(
- SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get
- ) {
- checkEvaluation(cast(cast(nts, TimestampType, UTC_OPT), StringType, UTC_OPT), nts)
- checkEvaluation(
- cast(cast(ts, StringType, UTC_OPT), TimestampType, UTC_OPT),
- fromJavaTimestamp(ts))
-
- // all convert to string type to check
- checkEvaluation(
- cast(cast(cast(nts, TimestampType, UTC_OPT), DateType, UTC_OPT), StringType),
- sd)
- checkEvaluation(
- cast(cast(cast(ts, DateType, UTC_OPT), TimestampType, UTC_OPT), StringType, UTC_OPT),
- zts)
- }
-
- checkEvaluation(cast(cast("abdef", BinaryType), StringType), "abdef")
-
- checkEvaluation(
- cast(
- cast(cast(cast(cast(cast("5", ByteType), ShortType), IntegerType), FloatType), DoubleType),
- LongType),
- 5.toLong)
-
- checkEvaluation(cast("23", DoubleType), 23d)
- checkEvaluation(cast("23", IntegerType), 23)
- checkEvaluation(cast("23", FloatType), 23f)
- checkEvaluation(cast("23", DecimalType.USER_DEFAULT), Decimal(23))
- checkEvaluation(cast("23", ByteType), 23.toByte)
- checkEvaluation(cast("23", ShortType), 23.toShort)
- checkEvaluation(cast(123, IntegerType), 123)
-
- checkEvaluation(cast(Literal.create(null, IntegerType), ShortType), null)
- }
-
- test("cast from boolean to timestamp") {
- val tsTrue = new Timestamp(0)
- tsTrue.setNanos(1000)
-
- val tsFalse = new Timestamp(0)
-
- checkEvaluation(cast(true, TimestampType), tsTrue)
-
- checkEvaluation(cast(false, TimestampType), tsFalse)
- }
-
- testGluten("cast string to timestamp") {
- DebuggableThreadUtils.parmap(
- ALL_TIMEZONES
- .filterNot(_.getId.contains("SystemV"))
- .filterNot(_.getId.contains("Europe/Kyiv"))
- .filterNot(_.getId.contains("America/Ciudad_Juarez"))
- .filterNot(_.getId.contains("Antarctica/Vostok"))
- .filterNot(_.getId.contains("Pacific/Kanton"))
- .filterNot(_.getId.contains("Asia/Tehran"))
- .filterNot(_.getId.contains("Iran")),
- prefix = "CastSuiteBase-cast-string-to-timestamp",
- maxThreads = 1
- ) {
- zid =>
- withSQLConf(
- SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId
- ) {
- def checkCastStringToTimestamp(str: String, expected: Timestamp): Unit = {
- checkEvaluation(cast(Literal(str), TimestampType, Option(zid.getId)), expected)
- }
-
- val tz = TimeZone.getTimeZone(zid)
- var c = Calendar.getInstance(tz)
- c.set(2015, 0, 1, 0, 0, 0)
- c.set(Calendar.MILLISECOND, 0)
- checkCastStringToTimestamp("2015", new Timestamp(c.getTimeInMillis))
- c = Calendar.getInstance(tz)
- c.set(2015, 2, 1, 0, 0, 0)
- c.set(Calendar.MILLISECOND, 0)
- checkCastStringToTimestamp("2015-03", new Timestamp(c.getTimeInMillis))
- c = Calendar.getInstance(tz)
- c.set(2015, 2, 18, 0, 0, 0)
- c.set(Calendar.MILLISECOND, 0)
- checkCastStringToTimestamp("2015-03-18", new Timestamp(c.getTimeInMillis))
- checkCastStringToTimestamp("2015-03-18 ", new Timestamp(c.getTimeInMillis))
-
- c = Calendar.getInstance(tz)
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 0)
- checkCastStringToTimestamp("2015-03-18 12:03:17", new Timestamp(c.getTimeInMillis))
- checkCastStringToTimestamp("2015-03-18T12:03:17", new Timestamp(c.getTimeInMillis))
-
- // If the string value includes timezone string, it represents the timestamp string
- // in the timezone regardless of the timeZoneId parameter.
- c = Calendar.getInstance(TimeZone.getTimeZone(UTC))
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 0)
- checkCastStringToTimestamp("2015-03-18T12:03:17Z", new Timestamp(c.getTimeInMillis))
- checkCastStringToTimestamp("2015-03-18 12:03:17Z", new Timestamp(c.getTimeInMillis))
-
- c = Calendar.getInstance(TimeZone.getTimeZone("GMT-01:00"))
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 0)
- // Unsupported timezone format for Velox backend.
- // checkCastStringToTimestamp("2015-03-18T12:03:17-1:0", new Timestamp(c.getTimeInMillis))
- checkCastStringToTimestamp("2015-03-18T12:03:17-01:00", new Timestamp(c.getTimeInMillis))
-
- c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30"))
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 0)
- checkCastStringToTimestamp("2015-03-18T12:03:17+07:30", new Timestamp(c.getTimeInMillis))
-
- c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:03"))
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 0)
- // Unsupported timezone format for Velox backend.
- // checkCastStringToTimestamp("2015-03-18T12:03:17+7:3",
- // new Timestamp(c.getTimeInMillis))
-
- // tests for the string including milliseconds.
- c = Calendar.getInstance(tz)
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 123)
- checkCastStringToTimestamp("2015-03-18 12:03:17.123", new Timestamp(c.getTimeInMillis))
- checkCastStringToTimestamp("2015-03-18T12:03:17.123", new Timestamp(c.getTimeInMillis))
-
- // If the string value includes timezone string, it represents the timestamp string
- // in the timezone regardless of the timeZoneId parameter.
- c = Calendar.getInstance(TimeZone.getTimeZone(UTC))
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 456)
- checkCastStringToTimestamp("2015-03-18T12:03:17.456Z", new Timestamp(c.getTimeInMillis))
- checkCastStringToTimestamp("2015-03-18 12:03:17.456Z", new Timestamp(c.getTimeInMillis))
-
- c = Calendar.getInstance(TimeZone.getTimeZone("GMT-01:00"))
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 123)
- // Unsupported timezone format for Velox backend.
- // checkCastStringToTimestamp("2015-03-18T12:03:17.123-1:0",
- // new Timestamp(c.getTimeInMillis))
- checkCastStringToTimestamp(
- "2015-03-18T12:03:17.123-01:00",
- new Timestamp(c.getTimeInMillis))
-
- c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30"))
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 123)
- checkCastStringToTimestamp(
- "2015-03-18T12:03:17.123+07:30",
- new Timestamp(c.getTimeInMillis))
-
- c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:03"))
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 123)
- // Unsupported timezone format for Velox backend.
- // checkCastStringToTimestamp("2015-03-18T12:03:17.123+7:3",
- // new Timestamp(c.getTimeInMillis))
- }
- }
- }
-
- testGluten("cast decimal to timestamp") {
- val tz = TimeZone.getTimeZone(TimeZone.getDefault.getID)
- val c = Calendar.getInstance(tz)
- c.set(2015, 0, 1, 0, 0, 0)
- c.set(Calendar.MILLISECOND, 123)
- val d = Decimal(c.getTimeInMillis.toDouble / 1000)
- checkEvaluation(cast(d, TimestampType), new Timestamp(c.getTimeInMillis))
- }
}
diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index df973d55c0..46383ebf0f 100644
--- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -107,16 +107,6 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenBindReferencesSuite]
enableSuite[GlutenBitwiseExpressionsSuite]
enableSuite[GlutenCastWithAnsiOffSuite]
- .exclude(
- "Process Infinity, -Infinity, NaN in case insensitive manner" // +inf not supported in folly.
- )
- // Set timezone through config.
- .exclude("data type casting")
- // Revised by setting timezone through config and commented unsupported cases.
- .exclude("cast string to timestamp")
- .exclude("cast from timestamp II")
- .exclude("SPARK-36286: invalid string cast to timestamp")
- .exclude("SPARK-39749: cast Decimal to string")
enableSuite[GlutenTryCastSuite]
.exclude(
"Process Infinity, -Infinity, NaN in case insensitive manner" // +inf not supported in folly.
diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala
index efde3f31e0..1e2986c004 100644
--- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala
+++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala
@@ -16,15 +16,8 @@
*/
package org.apache.spark.sql.catalyst.expressions
-import org.apache.spark.sql.GlutenTestsTrait
-import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, ALL_TIMEZONES, UTC, UTC_OPT}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp, millisToMicros, TimeZoneUTC}
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types._
-import org.apache.spark.util.DebuggableThreadUtils
-
-import java.sql.{Date, Timestamp}
-import java.util.{Calendar, TimeZone}
+import org.apache.spark.sql.shim.GlutenTestsTrait
class GlutenCastWithAnsiOffSuite extends CastWithAnsiOffSuite with GlutenTestsTrait {
override def beforeAll(): Unit = {
@@ -35,269 +28,4 @@ class GlutenCastWithAnsiOffSuite extends CastWithAnsiOffSuite with GlutenTestsTr
// In Vanilla spark, the checkEvaluation method doesn't invoke RowEncoder.
conf.setConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO, true)
}
-
- override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] = None): Cast = {
- v match {
- case lit: Expression =>
- logDebug(s"Cast from: ${lit.dataType.typeName}, to: ${targetType.typeName}")
- Cast(lit, targetType, timeZoneId)
- case _ =>
- val lit = Literal(v)
- logDebug(s"Cast from: ${lit.dataType.typeName}, to: ${targetType.typeName}")
- Cast(lit, targetType, timeZoneId)
- }
- }
-
- // Register UDT For test("SPARK-32828")
- UDTRegistration.register(classOf[IExampleBaseType].getName, classOf[ExampleBaseTypeUDT].getName)
- UDTRegistration.register(classOf[IExampleSubType].getName, classOf[ExampleSubTypeUDT].getName)
-
- testGluten("missing cases - from boolean") {
- (DataTypeTestUtils.numericTypeWithoutDecimal ++ Set(BooleanType)).foreach {
- t =>
- t match {
- case BooleanType =>
- checkEvaluation(cast(cast(true, BooleanType), t), true)
- checkEvaluation(cast(cast(false, BooleanType), t), false)
- case _ =>
- checkEvaluation(cast(cast(true, BooleanType), t), 1)
- checkEvaluation(cast(cast(false, BooleanType), t), 0)
- }
- }
- }
-
- testGluten("missing cases - from byte") {
- DataTypeTestUtils.numericTypeWithoutDecimal.foreach {
- t =>
- checkEvaluation(cast(cast(0, ByteType), t), 0)
- checkEvaluation(cast(cast(-1, ByteType), t), -1)
- checkEvaluation(cast(cast(1, ByteType), t), 1)
- }
- }
-
- testGluten("missing cases - from short") {
- DataTypeTestUtils.numericTypeWithoutDecimal.foreach {
- t =>
- checkEvaluation(cast(cast(0, ShortType), t), 0)
- checkEvaluation(cast(cast(-1, ShortType), t), -1)
- checkEvaluation(cast(cast(1, ShortType), t), 1)
- }
- }
-
- testGluten("missing cases - date self check") {
- val d = Date.valueOf("1970-01-01")
- checkEvaluation(cast(d, DateType), d)
- }
-
- testGluten("data type casting") {
- val sd = "1970-01-01"
- val d = Date.valueOf(sd)
- val zts = sd + " 00:00:00"
- val sts = sd + " 00:00:02"
- val nts = sts + ".1"
- val ts = withDefaultTimeZone(UTC)(Timestamp.valueOf(nts))
-
- // SystemV timezones are a legacy way of specifying timezones in Unix-like OS.
- // It is not supported by Velox.
- for (
- tz <- ALL_TIMEZONES
- .filterNot(_.getId.contains("SystemV"))
- .filterNot(_.getId.contains("America/Coyhaique"))
- ) {
- withSQLConf(
- SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz.getId
- ) {
- val timeZoneId = Option(tz.getId)
- var c = Calendar.getInstance(TimeZoneUTC)
- c.set(2015, 2, 8, 2, 30, 0)
- checkEvaluation(
- cast(
- cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
- TimestampType,
- timeZoneId),
- millisToMicros(c.getTimeInMillis))
- c = Calendar.getInstance(TimeZoneUTC)
- c.set(2015, 10, 1, 2, 30, 0)
- checkEvaluation(
- cast(
- cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
- TimestampType,
- timeZoneId),
- millisToMicros(c.getTimeInMillis))
- }
- }
-
- checkEvaluation(cast("abdef", StringType), "abdef")
- checkEvaluation(cast("12.65", DecimalType.SYSTEM_DEFAULT), Decimal(12.65))
-
- checkEvaluation(cast(cast(sd, DateType), StringType), sd)
- checkEvaluation(cast(cast(d, StringType), DateType), 0)
-
- withSQLConf(
- SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get
- ) {
- checkEvaluation(cast(cast(nts, TimestampType, UTC_OPT), StringType, UTC_OPT), nts)
- checkEvaluation(
- cast(cast(ts, StringType, UTC_OPT), TimestampType, UTC_OPT),
- fromJavaTimestamp(ts))
-
- // all convert to string type to check
- checkEvaluation(
- cast(cast(cast(nts, TimestampType, UTC_OPT), DateType, UTC_OPT), StringType),
- sd)
- checkEvaluation(
- cast(cast(cast(ts, DateType, UTC_OPT), TimestampType, UTC_OPT), StringType, UTC_OPT),
- zts)
- }
-
- checkEvaluation(cast(cast("abdef", BinaryType), StringType), "abdef")
-
- checkEvaluation(
- cast(
- cast(cast(cast(cast(cast("5", ByteType), ShortType), IntegerType), FloatType), DoubleType),
- LongType),
- 5.toLong)
-
- checkEvaluation(cast("23", DoubleType), 23d)
- checkEvaluation(cast("23", IntegerType), 23)
- checkEvaluation(cast("23", FloatType), 23f)
- checkEvaluation(cast("23", DecimalType.USER_DEFAULT), Decimal(23))
- checkEvaluation(cast("23", ByteType), 23.toByte)
- checkEvaluation(cast("23", ShortType), 23.toShort)
- checkEvaluation(cast(123, IntegerType), 123)
-
- checkEvaluation(cast(Literal.create(null, IntegerType), ShortType), null)
- }
-
- test("cast from boolean to timestamp") {
- val tsTrue = new Timestamp(0)
- tsTrue.setNanos(1000)
-
- val tsFalse = new Timestamp(0)
-
- checkEvaluation(cast(true, TimestampType), tsTrue)
-
- checkEvaluation(cast(false, TimestampType), tsFalse)
- }
-
- testGluten("cast string to timestamp") {
- DebuggableThreadUtils.parmap(
- ALL_TIMEZONES
- .filterNot(_.getId.contains("SystemV"))
- .filterNot(_.getId.contains("Europe/Kyiv"))
- .filterNot(_.getId.contains("America/Ciudad_Juarez"))
- .filterNot(_.getId.contains("Antarctica/Vostok"))
- .filterNot(_.getId.contains("Pacific/Kanton"))
- .filterNot(_.getId.contains("America/Coyhaique"))
- .filterNot(_.getId.contains("Asia/Tehran"))
- .filterNot(_.getId.contains("Iran")),
- prefix = "CastSuiteBase-cast-string-to-timestamp",
- maxThreads = 1
- ) {
- zid =>
- withSQLConf(
- SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId
- ) {
- def checkCastStringToTimestamp(str: String, expected: Timestamp): Unit = {
- checkEvaluation(cast(Literal(str), TimestampType, Option(zid.getId)), expected)
- }
-
- val tz = TimeZone.getTimeZone(zid)
- var c = Calendar.getInstance(tz)
- c.set(2015, 0, 1, 0, 0, 0)
- c.set(Calendar.MILLISECOND, 0)
- checkCastStringToTimestamp("2015", new Timestamp(c.getTimeInMillis))
- c = Calendar.getInstance(tz)
- c.set(2015, 2, 1, 0, 0, 0)
- c.set(Calendar.MILLISECOND, 0)
- checkCastStringToTimestamp("2015-03", new Timestamp(c.getTimeInMillis))
- c = Calendar.getInstance(tz)
- c.set(2015, 2, 18, 0, 0, 0)
- c.set(Calendar.MILLISECOND, 0)
- checkCastStringToTimestamp("2015-03-18", new Timestamp(c.getTimeInMillis))
- checkCastStringToTimestamp("2015-03-18 ", new Timestamp(c.getTimeInMillis))
-
- c = Calendar.getInstance(tz)
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 0)
- checkCastStringToTimestamp("2015-03-18 12:03:17", new Timestamp(c.getTimeInMillis))
- checkCastStringToTimestamp("2015-03-18T12:03:17", new Timestamp(c.getTimeInMillis))
-
- // If the string value includes timezone string, it represents the timestamp string
- // in the timezone regardless of the timeZoneId parameter.
- c = Calendar.getInstance(TimeZone.getTimeZone(UTC))
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 0)
- checkCastStringToTimestamp("2015-03-18T12:03:17Z", new Timestamp(c.getTimeInMillis))
- checkCastStringToTimestamp("2015-03-18 12:03:17Z", new Timestamp(c.getTimeInMillis))
-
- c = Calendar.getInstance(TimeZone.getTimeZone("GMT-01:00"))
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 0)
- // Unsupported timezone format for Velox backend.
- // checkCastStringToTimestamp("2015-03-18T12:03:17-1:0", new Timestamp(c.getTimeInMillis))
- checkCastStringToTimestamp("2015-03-18T12:03:17-01:00", new Timestamp(c.getTimeInMillis))
-
- c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30"))
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 0)
- checkCastStringToTimestamp("2015-03-18T12:03:17+07:30", new Timestamp(c.getTimeInMillis))
-
- c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:03"))
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 0)
- // Unsupported timezone format for Velox backend.
- // checkCastStringToTimestamp("2015-03-18T12:03:17+7:3",
- // new Timestamp(c.getTimeInMillis))
-
- // tests for the string including milliseconds.
- c = Calendar.getInstance(tz)
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 123)
- checkCastStringToTimestamp("2015-03-18 12:03:17.123", new Timestamp(c.getTimeInMillis))
- checkCastStringToTimestamp("2015-03-18T12:03:17.123", new Timestamp(c.getTimeInMillis))
-
- // If the string value includes timezone string, it represents the timestamp string
- // in the timezone regardless of the timeZoneId parameter.
- c = Calendar.getInstance(TimeZone.getTimeZone(UTC))
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 456)
- checkCastStringToTimestamp("2015-03-18T12:03:17.456Z", new Timestamp(c.getTimeInMillis))
- checkCastStringToTimestamp("2015-03-18 12:03:17.456Z", new Timestamp(c.getTimeInMillis))
-
- c = Calendar.getInstance(TimeZone.getTimeZone("GMT-01:00"))
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 123)
- // Unsupported timezone format for Velox backend.
- // checkCastStringToTimestamp("2015-03-18T12:03:17.123-1:0",
- // new Timestamp(c.getTimeInMillis))
- checkCastStringToTimestamp(
- "2015-03-18T12:03:17.123-01:00",
- new Timestamp(c.getTimeInMillis))
-
- c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30"))
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 123)
- checkCastStringToTimestamp(
- "2015-03-18T12:03:17.123+07:30",
- new Timestamp(c.getTimeInMillis))
-
- c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:03"))
- c.set(2015, 2, 18, 12, 3, 17)
- c.set(Calendar.MILLISECOND, 123)
- // Unsupported timezone format for Velox backend.
- // checkCastStringToTimestamp("2015-03-18T12:03:17.123+7:3",
- // new Timestamp(c.getTimeInMillis))
- }
- }
- }
-
- testGluten("cast decimal to timestamp") {
- val tz = TimeZone.getTimeZone(TimeZone.getDefault.getID)
- val c = Calendar.getInstance(tz)
- c.set(2015, 0, 1, 0, 0, 0)
- c.set(Calendar.MILLISECOND, 123)
- val d = Decimal(c.getTimeInMillis.toDouble / 1000)
- checkEvaluation(cast(d, TimestampType), new Timestamp(c.getTimeInMillis))
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org