You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/11 18:11:52 UTC
(datafusion-comet) branch gh-readonly-queue/main/pr-6841-d8d37fc527e9498dcab5c1a52a470edb8fa11718 created (now 34e29a1690)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-6841-d8d37fc527e9498dcab5c1a52a470edb8fa11718
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
at 34e29a1690 fix: accept all BMP Unicode decimal digits in cast(string as decimal) (#6841)
This branch includes the following new commits:
new 34e29a1690 fix: accept all BMP Unicode decimal digits in cast(string as decimal) (#6841)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion-comet) 01/01: fix: accept all BMP Unicode decimal digits in cast(string as decimal) (#6841)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6841-d8d37fc527e9498dcab5c1a52a470edb8fa11718
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit 34e29a16902aecec0aa068ba5c5172f3c10b060e
Author: Liang-Chi Hsieh <vi...@gmail.com>
AuthorDate: Sun Oct 11 18:11:30 2026 +0000
fix: accept all BMP Unicode decimal digits in cast(string as decimal) (#6841)
* fix: accept all BMP Unicode decimal digits in cast(string as decimal)
Spark casts a string to decimal with `new java.math.BigDecimal(s.trim)`,
which accepts any char for which `Character.isDigit(char)` is true, in
both the mantissa and the exponent. Comet only normalized fullwidth
digits, so strings such as '٣' (U+0663 ARABIC-INDIC DIGIT THREE) or
'1e३' returned NULL, or raised CAST_INVALID_INPUT under ANSI, where Spark
returns a value.
Normalize every non-ASCII Nd digit in the Basic Multilingual Plane to
ASCII before parsing. Supplementary-plane digits stay rejected, since
BigDecimal sees them as surrogate pairs. The pure-ASCII fast path is
unchanged. Integral, floating-point and date casts are left alone: Spark
only accepts ASCII digits there.
* perf: keep ASCII and fullwidth digits off the table lookup
Normalizing every char through the Nd table slowed down already
supported fullwidth and mixed ASCII/fullwidth input. Scan bytes instead,
copying ASCII directly and mapping the fullwidth UTF-8 encoding in place
as before, and only decode a char for the table lookup otherwise. The
output buffer is preallocated, since it is never longer than the input.
Add fullwidth, mixed ASCII/fullwidth and Arabic-Indic cases to the
cast_from_string benchmark.
---
.../expressions/_category_template/cast.md | 7 +-
native/spark-expr/benches/cast_from_string.rs | 46 +++++
native/spark-expr/src/conversion_funcs/string.rs | 200 ++++++++++++++++++---
.../cast/cast_string_to_decimal_unicode_digits.sql | 88 +++++++++
.../cast_string_to_decimal_unicode_digits_ansi.sql | 76 ++++++++
5 files changed, 393 insertions(+), 24 deletions(-)
diff --git a/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md b/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md
index c023539f36..f55199125c 100644
--- a/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md
+++ b/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md
@@ -64,8 +64,11 @@ including:
`java.lang.String.trim` that Spark applies before handing the value to `BigDecimal`.
- Null bytes (`\u0000`) at the start or end of a string are therefore trimmed. Null bytes
embedded in the middle of a string produce `NULL`.
-- Fullwidth Unicode digits (U+FF10–U+FF19, e.g. `１２３.４５`) are treated as their ASCII
- equivalents, so `CAST('１２３.４５' AS DECIMAL(10,2))` returns `123.45`.
+- Non-ASCII decimal digits that Java's `BigDecimal` accepts, i.e. every Unicode `Nd` digit in
+ the Basic Multilingual Plane (fullwidth `１２３.４５`, Arabic-Indic `١٢٣`, Devanagari `१२३`,
+ ...), are treated as their ASCII equivalents in both the mantissa and the exponent, so
+ `CAST('١٢٣.٤٥' AS DECIMAL(10,2))` returns `123.45`. Digits outside the Basic Multilingual
+ Plane, such as U+1D7CE MATHEMATICAL BOLD DIGIT ZERO, are rejected as they are by Spark.
- Scientific notation (e.g. `1.23E+5`) is supported.
- Special values (`inf`, `infinity`, `nan`) produce `NULL`.
diff --git a/native/spark-expr/benches/cast_from_string.rs b/native/spark-expr/benches/cast_from_string.rs
index cd34467012..608fe75469 100644
--- a/native/spark-expr/benches/cast_from_string.rs
+++ b/native/spark-expr/benches/cast_from_string.rs
@@ -136,6 +136,29 @@ fn criterion_benchmark(c: &mut Criterion) {
group.finish();
}
+ // str -> decimal with non-ASCII digits, which take the normalization path in
+ // `parse_string_to_decimal` instead of the pure-ASCII fast path
+ let spark_cast_options = SparkCastOptions::new(EvalMode::Legacy, "", false);
+ let cast = Cast::new(
+ expr.clone(),
+ DataType::Decimal128(38, 10),
+ spark_cast_options,
+ None,
+ None,
+ );
+ let mut group = c.benchmark_group("cast_string_to_decimal_unicode/legacy");
+ for (name, zero, every_other) in [
+ ("fullwidth", '\u{FF10}', false),
+ ("mixed_ascii_fullwidth", '\u{FF10}', true),
+ ("arabic_indic", '\u{0660}', false),
+ ] {
+ let batch = create_unicode_decimal_cast_string_batch(zero, every_other);
+ group.bench_function(name, |b| {
+ b.iter(|| cast.evaluate(&batch).unwrap());
+ });
+ }
+ group.finish();
+
// str -> boolean and str -> float benchmarks, with and without the leading/trailing
// whitespace that exercises the trim helpers in `conversion_funcs::trim`
let bool_batch = create_boolean_string_batch(false);
@@ -252,6 +275,29 @@ fn create_decimal_cast_string_batch() -> RecordBatch {
})
}
+/// Create batch of decimal strings whose digits are re-spelled in the script whose zero is
+/// `zero`, either all of them or every other one (mixing with ASCII)
+fn create_unicode_decimal_cast_string_batch(zero: char, every_other: bool) -> RecordBatch {
+ let mut rng = StdRng::seed_from_u64(42);
+ string_batch(8192, 10, move |_| {
+ let ascii = format!(
+ "-{}.{}",
+ rng.random_range(0..1_000_000_000u32),
+ rng.random_range(0..1_000_000_000u32)
+ );
+ ascii
+ .chars()
+ .enumerate()
+ .map(|(i, c)| match c.to_digit(10) {
+ Some(d) if !every_other || i % 2 == 0 => {
+ char::from_u32(u32::from(zero) + d).unwrap()
+ }
+ _ => c,
+ })
+ .collect()
+ })
+}
+
fn config() -> Criterion {
Criterion::default()
}
diff --git a/native/spark-expr/src/conversion_funcs/string.rs b/native/spark-expr/src/conversion_funcs/string.rs
index f278bafda6..89c18255d0 100644
--- a/native/spark-expr/src/conversion_funcs/string.rs
+++ b/native/spark-expr/src/conversion_funcs/string.rs
@@ -461,37 +461,110 @@ fn cast_string_to_decimal256_impl(
))
}
-/// Normalize fullwidth Unicode digits (U+FF10–U+FF19) to their ASCII equivalents.
+/// Code point of the zero of each run of ten non-ASCII decimal digits (Unicode general
+/// category `Nd`) in the Basic Multilingual Plane, in ascending order. Every run is contiguous,
+/// so the digit at `zero + n` has value `n`.
///
-/// Spark's UTF8String parser treats fullwidth digits as numerically equivalent to
-/// ASCII digits, e.g. "１２３.４５" parses as 123.45. Each fullwidth digit encodes
-/// to exactly three UTF-8 bytes: [0xEF, 0xBC, 0x90+n] for digit n. The ASCII
-/// equivalent is 0x30+n, so the conversion is: third_byte - 0x60.
+/// This is the set of `char`s for which Java's `Character.isDigit(char)` is true, minus ASCII.
+/// It is identical under JDK 17 (Unicode 13) and JDK 21 (Unicode 15); all `Nd` digits added
+/// since lie outside the BMP.
+const BMP_NON_ASCII_DIGIT_ZEROS: [u16; 36] = [
+ 0x0660, // Arabic-Indic
+ 0x06F0, // Extended Arabic-Indic
+ 0x07C0, // NKo
+ 0x0966, // Devanagari
+ 0x09E6, // Bengali
+ 0x0A66, // Gurmukhi
+ 0x0AE6, // Gujarati
+ 0x0B66, // Oriya
+ 0x0BE6, // Tamil
+ 0x0C66, // Telugu
+ 0x0CE6, // Kannada
+ 0x0D66, // Malayalam
+ 0x0DE6, // Sinhala Lith
+ 0x0E50, // Thai
+ 0x0ED0, // Lao
+ 0x0F20, // Tibetan
+ 0x1040, // Myanmar
+ 0x1090, // Myanmar Shan
+ 0x17E0, // Khmer
+ 0x1810, // Mongolian
+ 0x1946, // Limbu
+ 0x19D0, // New Tai Lue
+ 0x1A80, // Tai Tham Hora
+ 0x1A90, // Tai Tham Tham
+ 0x1B50, // Balinese
+ 0x1BB0, // Sundanese
+ 0x1C40, // Lepcha
+ 0x1C50, // Ol Chiki
+ 0xA620, // Vai
+ 0xA8D0, // Saurashtra
+ 0xA900, // Kayah Li
+ 0xA9D0, // Javanese
+ 0xA9F0, // Myanmar Tai Laing
+ 0xAA50, // Cham
+ 0xABF0, // Meetei Mayek
+ 0xFF10, // Fullwidth
+];
+
+/// The value of `c` if it is a non-ASCII decimal digit that Java's `BigDecimal` accepts.
///
-/// All other bytes (ASCII or other multi-byte sequences) are passed through
-/// unchanged, so the output is valid UTF-8 whenever the input is.
-fn normalize_fullwidth_digits(s: &str) -> String {
- let bytes = s.as_bytes();
+/// `BigDecimal` scans UTF-16 code units and tests each with `Character.isDigit(char)`, so only
+/// BMP digits count: a supplementary digit such as U+1D7CE MATHEMATICAL BOLD DIGIT ZERO is a
+/// surrogate pair, neither half of which is a digit, and the parse fails.
+#[inline]
+fn non_ascii_digit_value(c: char) -> Option<u8> {
+ let cp = u32::from(c);
+ if cp > 0xFFFF {
+ return None;
+ }
+ let idx = BMP_NON_ASCII_DIGIT_ZEROS.partition_point(|&zero| u32::from(zero) <= cp);
+ let zero = u32::from(BMP_NON_ASCII_DIGIT_ZEROS[idx.checked_sub(1)?]);
+ let value = cp - zero;
+ (value < 10).then_some(value as u8)
+}
+
+/// Replace every non-ASCII decimal digit that Java's `BigDecimal` accepts with its ASCII
+/// equivalent, e.g. "١٢٣.٤٥" and "１２３.４５" both become "123.45".
+///
+/// Spark parses decimals with `new java.math.BigDecimal(str)`, which reads each digit through
+/// `Character.digit(c, 10)` in both the mantissa and the exponent, so digits from any script
+/// can be mixed freely ("1٢e३" is 12000). All other characters are kept unchanged and are
+/// rejected later by [`parse_decimal_str`].
+fn normalize_unicode_digits(s: &str) -> String {
+ // Every non-ASCII digit is at least two UTF-8 bytes and becomes one, so the output is
+ // never longer than the input.
let mut out = Vec::with_capacity(s.len());
+ let bytes = s.as_bytes();
let mut i = 0;
while i < bytes.len() {
- if i + 2 < bytes.len()
- && bytes[i] == 0xEF
- && bytes[i + 1] == 0xBC
- && bytes[i + 2] >= 0x90
- && bytes[i + 2] <= 0x99
+ let b = bytes[i];
+ // ASCII (including mixed ASCII/fullwidth input) and fullwidth digits, encoded as
+ // [0xEF, 0xBC, 0x90 + n], are the common cases, so they skip the table lookup.
+ if b.is_ascii() {
+ out.push(b);
+ i += 1;
+ } else if b == 0xEF
+ && bytes.get(i + 1) == Some(&0xBC)
+ && matches!(bytes.get(i + 2), Some(0x90..=0x99))
{
// e.g. 0x91 - 0x60 = 0x31 = b'1'
out.push(bytes[i + 2] - 0x60);
i += 3;
} else {
- out.push(bytes[i]);
- i += 1;
+ // `i` is on a char boundary: every branch above advances by a whole char.
+ let mut rest = s[i..].chars();
+ let c = rest.next().unwrap();
+ let next = bytes.len() - rest.as_str().len();
+ match non_ascii_digit_value(c) {
+ Some(d) => out.push(b'0' + d),
+ None => out.extend_from_slice(&bytes[i..next]),
+ }
+ i = next;
}
}
- // SAFETY: we only replace valid 3-byte UTF-8 sequences [EF BC 9X] with a
- // single ASCII byte; all other bytes are copied unchanged, preserving the
- // UTF-8 invariant of the input.
+ // SAFETY: every branch either copies a whole UTF-8 encoded char unchanged or replaces it
+ // with a single ASCII byte, so the output is valid UTF-8 because the input is.
unsafe { String::from_utf8_unchecked(out) }
}
@@ -585,13 +658,13 @@ fn parse_string_to_decimal(input_str: &str, precision: u8, scale: i8) -> SparkRe
// producing NULL.
let trimmed = trim_java_string(input_str);
- // Normalize fullwidth digits to ASCII. Fast path skips the allocation for
- // pure-ASCII strings, which is the common case.
+ // Normalize non-ASCII digits (Arabic-Indic, Devanagari, fullwidth, ...) to ASCII. Fast path
+ // skips the allocation for pure-ASCII strings, which is the common case.
let normalized;
let trimmed = if trimmed.is_ascii() {
trimmed
} else {
- normalized = normalize_fullwidth_digits(trimmed);
+ normalized = normalize_unicode_digits(trimmed);
normalized.as_str()
};
@@ -2335,6 +2408,89 @@ mod tests {
assert!(parse_string_to_decimal(&over_long, 38, 10).is_err());
}
+ #[test]
+ fn test_non_ascii_digit_value() {
+ for &zero in BMP_NON_ASCII_DIGIT_ZEROS.iter() {
+ for n in 0..10u8 {
+ let c = char::from_u32(u32::from(zero) + u32::from(n)).unwrap();
+ assert_eq!(non_ascii_digit_value(c), Some(n), "{c:?}");
+ }
+ // The code point just past each run is not a digit of that run.
+ let after = char::from_u32(u32::from(zero) + 10).unwrap();
+ assert_eq!(non_ascii_digit_value(after), None, "{after:?}");
+ }
+ for c in [
+ '0',
+ '9',
+ 'a',
+ '.',
+ '\u{065F}', // just before Arabic-Indic zero
+ '\u{00B2}', // SUPERSCRIPT TWO (No)
+ '\u{2163}', // ROMAN NUMERAL FOUR (Nl)
+ '\u{2460}', // CIRCLED DIGIT ONE (No)
+ '\u{066B}', // ARABIC DECIMAL SEPARATOR
+ '\u{FFFF}',
+ '\u{1D7CE}', // MATHEMATICAL BOLD DIGIT ZERO: Nd, but outside the BMP
+ '\u{104A3}', // OSMANYA DIGIT THREE: Nd, but outside the BMP
+ ] {
+ assert_eq!(non_ascii_digit_value(c), None, "{c:?}");
+ }
+ }
+
+ #[test]
+ fn test_parse_string_to_decimal_unicode_digits() {
+ // Expected values come from `new java.math.BigDecimal(s)` on JDK 17, which is what
+ // Spark's Decimal.fromString / fromStringANSI call.
+ let ok = |s: &str, precision: u8, scale: i8| {
+ parse_string_to_decimal(s, precision, scale)
+ .unwrap()
+ .unwrap()
+ };
+ assert_eq!(ok("٣", 10, 2), 300); // Arabic-Indic
+ assert_eq!(ok("۳", 10, 2), 300); // Extended Arabic-Indic
+ assert_eq!(ok("३", 10, 2), 300); // Devanagari
+ assert_eq!(ok("৩", 10, 2), 300); // Bengali
+ assert_eq!(ok("๓", 10, 2), 300); // Thai
+ assert_eq!(ok("߃", 10, 2), 300); // NKo
+ assert_eq!(ok("꧓", 10, 2), 300); // Javanese
+ assert_eq!(ok("１２３.４５", 10, 2), 12345); // Fullwidth
+ assert_eq!(ok("1٣", 10, 2), 1300);
+ assert_eq!(ok("-٣", 10, 2), -300);
+ assert_eq!(ok("+٣.٣٣", 10, 2), 333);
+ assert_eq!(ok(".٣", 10, 2), 30);
+ assert_eq!(ok("١٢٣.٤٥٥", 10, 2), 12346); // HALF_UP rounding
+ assert_eq!(ok("1e٣", 10, 2), 100000); // non-ASCII exponent digits
+ assert_eq!(ok("1E-٣", 10, 3), 1);
+ assert_eq!(ok("1e+٣", 10, 2), 100000);
+ assert_eq!(ok("1٢e३", 10, 0), 12000); // mixed scripts
+ assert_eq!(ok(" \t٣\n", 10, 2), 300); // ASCII trim still applies
+ assert_eq!(ok(&"٣".repeat(20), 38, 0), 33333333333333333333);
+
+ // Rejected by BigDecimal: not Nd, or Nd outside the BMP.
+ for s in [
+ "\u{1D7D0}", // MATHEMATICAL BOLD DIGIT TWO
+ "1\u{1D7D0}",
+ "1e\u{1D7D0}",
+ "\u{2163}", // ROMAN NUMERAL FOUR
+ "\u{00B2}", // SUPERSCRIPT TWO
+ "١\u{066B}٥", // ARABIC DECIMAL SEPARATOR is not '.'
+ "\u{00A0}٣", // non-ASCII whitespace is not trimmed
+ ] {
+ // The error reports the original string, not the normalized one.
+ match parse_string_to_decimal(s, 10, 2) {
+ Err(SparkError::CastInvalidValue { value, .. }) => assert_eq!(value, s),
+ other => panic!("{s:?}: {other:?}"),
+ }
+ }
+
+ // Overflow is reported as such, not as an invalid input.
+ let err = parse_string_to_decimal("٣٣٣٣٣٣٣٣٣", 10, 2).unwrap_err();
+ assert!(
+ matches!(err, SparkError::NumericValueOutOfRange { .. }),
+ "{err}"
+ );
+ }
+
#[test]
#[cfg_attr(miri, ignore)] // test takes too long with miri
fn test_cast_string_to_timestamp() {
diff --git a/spark/src/test/resources/sql-tests/expressions/cast/cast_string_to_decimal_unicode_digits.sql b/spark/src/test/resources/sql-tests/expressions/cast/cast_string_to_decimal_unicode_digits.sql
new file mode 100644
index 0000000000..f8d22a1d55
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/cast/cast_string_to_decimal_unicode_digits.sql
@@ -0,0 +1,88 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- cast(string as decimal) with non-ASCII digits, non-ANSI mode. Spark parses the trimmed string
+-- with `new java.math.BigDecimal(s)`, which accepts every char for which
+-- `Character.isDigit(char)` is true: all Unicode `Nd` digits in the Basic Multilingual Plane,
+-- in the mantissa and in the exponent, mixed freely across scripts. Supplementary-plane digits
+-- (e.g. U+1D7D0 MATHEMATICAL BOLD DIGIT TWO) are surrogate pairs and are rejected.
+--
+-- The integral, floating-point and date casts of the same strings are ASCII-only in Spark
+-- (UTF8String.toLong, Double.parseDouble, SparkDateTimeUtils.stringToDate) and must stay NULL.
+
+-- Config: spark.sql.ansi.enabled=false
+
+statement
+CREATE TABLE test_cast_str_dec_unicode(id int, s string) USING parquet
+
+-- 1: U+0663 ARABIC-INDIC DIGIT THREE
+-- 2: U+06F3 EXTENDED ARABIC-INDIC DIGIT THREE
+-- 3: U+0969 DEVANAGARI DIGIT THREE
+-- 4: U+09E9 BENGALI DIGIT THREE
+-- 5: U+0E53 THAI DIGIT THREE
+-- 6: U+07C3 NKO DIGIT THREE
+-- 7: U+A9D3 JAVANESE DIGIT THREE
+-- 8: fullwidth digits
+-- 9-17: signs, '.', HALF_UP rounding, exponent digits, mixed scripts, ASCII trim
+-- 18: 20 Arabic-Indic digits, too many for the cast targets below
+-- 19-25: rejected by BigDecimal: supplementary-plane Nd digits, ROMAN NUMERAL FOUR (Nl),
+-- SUPERSCRIPT TWO (No), ARABIC DECIMAL SEPARATOR, leading NBSP (not trimmed)
+statement
+INSERT INTO test_cast_str_dec_unicode VALUES
+ (1, '٣'),
+ (2, '۳'),
+ (3, '३'),
+ (4, '৩'),
+ (5, '๓'),
+ (6, '߃'),
+ (7, '꧓'),
+ (8, '１２３.４５'),
+ (9, '1٣'),
+ (10, '-٣'),
+ (11, '+٣.٣٣'),
+ (12, '.٣'),
+ (13, '١٢٣.٤٥٥'),
+ (14, '1e٣'),
+ (15, '1E-٣'),
+ (16, '1٢e३'),
+ (17, concat(' ٣', chr(9))),
+ (18, '٣٣٣٣٣٣٣٣٣٣٣٣٣٣٣٣٣٣٣٣'),
+ (19, '𝟐'),
+ (20, '1𝟐'),
+ (21, '1e𝟐'),
+ (22, 'Ⅳ'),
+ (23, '²'),
+ (24, '١٫٥'),
+ (25, concat(chr(160), '٣')),
+ (26, NULL)
+
+query
+SELECT id, cast(s as decimal(10,2)), cast(s as decimal(38,10)), cast(s as decimal(5,0))
+FROM test_cast_str_dec_unicode ORDER BY id
+
+query
+SELECT id, try_cast(s as decimal(10,2)), try_cast(s as decimal(38,0))
+FROM test_cast_str_dec_unicode ORDER BY id
+
+-- Spark's integral, floating-point and date parsers only accept ASCII digits
+query
+SELECT id, cast(s as int), cast(s as bigint), cast(s as double), cast(s as date)
+FROM test_cast_str_dec_unicode ORDER BY id
+
+-- literal arguments
+query
+SELECT cast('٣' as decimal(10,2)), cast('१२३.४५' as decimal(10,2)), cast('𝟐' as decimal(10,2))
diff --git a/spark/src/test/resources/sql-tests/expressions/cast/cast_string_to_decimal_unicode_digits_ansi.sql b/spark/src/test/resources/sql-tests/expressions/cast/cast_string_to_decimal_unicode_digits_ansi.sql
new file mode 100644
index 0000000000..f546893dfa
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/cast/cast_string_to_decimal_unicode_digits_ansi.sql
@@ -0,0 +1,76 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- cast(string as decimal) with non-ASCII digits, ANSI mode. Spark parses the trimmed string
+-- with `new java.math.BigDecimal(s)`, which accepts every Unicode `Nd` digit in the Basic
+-- Multilingual Plane, so these must cast to a value rather than raise CAST_INVALID_INPUT.
+-- Supplementary-plane digits are rejected by BigDecimal and must still raise.
+
+-- Config: spark.sql.ansi.enabled=true
+
+statement
+CREATE TABLE test_cast_str_dec_unicode_ansi(id int, s string) USING parquet
+
+-- Arabic-Indic, Devanagari, Thai, fullwidth, signs, '.', rounding, exponent digits, mixed
+-- scripts and ASCII trim; every value fits decimal(10,2)
+statement
+INSERT INTO test_cast_str_dec_unicode_ansi VALUES
+ (1, '٣'),
+ (2, '३'),
+ (3, '๓'),
+ (4, '１２３.４５'),
+ (5, '-٣'),
+ (6, '+٣.٣٣'),
+ (7, '١٢٣.٤٥٥'),
+ (8, '1e٣'),
+ (9, '1E-٣'),
+ (10, '1٢e३'),
+ (11, concat(' ٣', chr(9))),
+ (12, NULL)
+
+statement
+CREATE TABLE test_cast_str_dec_unicode_ansi_bad(s string) USING parquet
+
+-- U+1D7D0 MATHEMATICAL BOLD DIGIT TWO is Nd but outside the BMP
+statement
+INSERT INTO test_cast_str_dec_unicode_ansi_bad VALUES ('𝟐')
+
+statement
+CREATE TABLE test_cast_str_dec_unicode_ansi_big(s string) USING parquet
+
+statement
+INSERT INTO test_cast_str_dec_unicode_ansi_big VALUES ('٣٣٣٣٣٣٣٣٣')
+
+query
+SELECT id, cast(s as decimal(10,2)), cast(s as decimal(38,10))
+FROM test_cast_str_dec_unicode_ansi ORDER BY id
+
+query
+SELECT id, try_cast(s as decimal(10,2)) FROM test_cast_str_dec_unicode_ansi ORDER BY id
+
+query expect_error(CAST_INVALID_INPUT)
+SELECT cast(s as decimal(10,2)) FROM test_cast_str_dec_unicode_ansi_bad
+
+-- the value is valid but does not fit decimal(10,2)
+query expect_error(NUMERIC_VALUE_OUT_OF_RANGE)
+SELECT cast(s as decimal(10,2)) FROM test_cast_str_dec_unicode_ansi_big
+
+-- try_cast suppresses both errors
+query
+SELECT try_cast(s as decimal(10,2)) FROM test_cast_str_dec_unicode_ansi_bad
+UNION ALL
+SELECT try_cast(s as decimal(10,2)) FROM test_cast_str_dec_unicode_ansi_big
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org