You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/11 16:58:31 UTC
(datafusion-comet) 01/01: fix: accept all BMP Unicode decimal digits in cast(string as decimal) (#6841)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6841-d8d37fc527e9498dcab5c1a52a470edb8fa11718
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit 04d66aecc24e239157ef2a7376e9d07ac996ca90
Author: Liang-Chi Hsieh <vi...@gmail.com>
AuthorDate: Sun Oct 11 16:57:45 2026 +0000
fix: accept all BMP Unicode decimal digits in cast(string as decimal) (#6841)
* fix: accept all BMP Unicode decimal digits in cast(string as decimal)
Spark casts a string to decimal with `new java.math.BigDecimal(s.trim)`,
which accepts any char for which `Character.isDigit(char)` is true, in
both the mantissa and the exponent. Comet only normalized fullwidth
digits, so strings such as '٣' (U+0663 ARABIC-INDIC DIGIT THREE) or
'1e३' returned NULL, or raised CAST_INVALID_INPUT under ANSI, where Spark
returns a value.
Normalize every non-ASCII Nd digit in the Basic Multilingual Plane to
ASCII before parsing. Supplementary-plane digits stay rejected, since
BigDecimal sees them as surrogate pairs. The pure-ASCII fast path is
unchanged. Integral, floating-point and date casts are left alone: Spark
only accepts ASCII digits there.
* perf: keep ASCII and fullwidth digits off the table lookup
Normalizing every char through the Nd table slowed down already
supported fullwidth and mixed ASCII/fullwidth input. Scan bytes instead,
copying ASCII directly and mapping the fullwidth UTF-8 encoding in place
as before, and only decode a char for the table lookup otherwise. The
output buffer is preallocated, since it is never longer than the input.
Add fullwidth, mixed ASCII/fullwidth and Arabic-Indic cases to the
cast_from_string benchmark.
---
.../expressions/_category_template/cast.md | 7 +-
native/spark-expr/benches/cast_from_string.rs | 46 +++++
native/spark-expr/src/conversion_funcs/string.rs | 200 ++++++++++++++++++---
.../cast/cast_string_to_decimal_unicode_digits.sql | 88 +++++++++
.../cast_string_to_decimal_unicode_digits_ansi.sql | 76 ++++++++
5 files changed, 393 insertions(+), 24 deletions(-)
diff --git a/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md b/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md
index c023539f36..f55199125c 100644
--- a/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md
+++ b/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md
@@ -64,8 +64,11 @@ including:
`java.lang.String.trim` that Spark applies before handing the value to `BigDecimal`.
- Null bytes (`\u0000`) at the start or end of a string are therefore trimmed. Null bytes
embedded in the middle of a string produce `NULL`.
-- Fullwidth Unicode digits (U+FF10–U+FF19, e.g. `１２３.４５`) are treated as their ASCII
- equivalents, so `CAST('１２３.４５' AS DECIMAL(10,2))` returns `123.45`.
+- Non-ASCII decimal digits that Java's `BigDecimal` accepts, i.e. every Unicode `Nd` digit in
+ the Basic Multilingual Plane (fullwidth `１２３.４５`, Arabic-Indic `١٢٣`, Devanagari `१२३`,
+ ...), are treated as their ASCII equivalents in both the mantissa and the exponent, so
+ `CAST('١٢٣.٤٥' AS DECIMAL(10,2))` returns `123.45`. Digits outside the Basic Multilingual
+ Plane, such as U+1D7CE MATHEMATICAL BOLD DIGIT ZERO, are rejected as they are by Spark.
- Scientific notation (e.g. `1.23E+5`) is supported.
- Special values (`inf`, `infinity`, `nan`) produce `NULL`.
diff --git a/native/spark-expr/benches/cast_from_string.rs b/native/spark-expr/benches/cast_from_string.rs
index cd34467012..608fe75469 100644
--- a/native/spark-expr/benches/cast_from_string.rs
+++ b/native/spark-expr/benches/cast_from_string.rs
@@ -136,6 +136,29 @@ fn criterion_benchmark(c: &mut Criterion) {
group.finish();
}
+ // str -> decimal with non-ASCII digits, which take the normalization path in
+ // `parse_string_to_decimal` instead of the pure-ASCII fast path
+ let spark_cast_options = SparkCastOptions::new(EvalMode::Legacy, "", false);
+ let cast = Cast::new(
+ expr.clone(),
+ DataType::Decimal128(38, 10),
+ spark_cast_options,
+ None,
+ None,
+ );
+ let mut group = c.benchmark_group("cast_string_to_decimal_unicode/legacy");
+ for (name, zero, every_other) in [
+ ("fullwidth", '\u{FF10}', false),
+ ("mixed_ascii_fullwidth", '\u{FF10}', true),
+ ("arabic_indic", '\u{0660}', false),
+ ] {
+ let batch = create_unicode_decimal_cast_string_batch(zero, every_other);
+ group.bench_function(name, |b| {
+ b.iter(|| cast.evaluate(&batch).unwrap());
+ });
+ }
+ group.finish();
+
// str -> boolean and str -> float benchmarks, with and without the leading/trailing
// whitespace that exercises the trim helpers in `conversion_funcs::trim`
let bool_batch = create_boolean_string_batch(false);
@@ -252,6 +275,29 @@ fn create_decimal_cast_string_batch() -> RecordBatch {
})
}
+/// Create batch of decimal strings whose digits are re-spelled in the script whose zero is
+/// `zero`, either all of them or every other one (mixing with ASCII)
+fn create_unicode_decimal_cast_string_batch(zero: char, every_other: bool) -> RecordBatch {
+ let mut rng = StdRng::seed_from_u64(42);
+ string_batch(8192, 10, move |_| {
+ let ascii = format!(
+ "-{}.{}",
+ rng.random_range(0..1_000_000_000u32),
+ rng.random_range(0..1_000_000_000u32)
+ );
+ ascii
+ .chars()
+ .enumerate()
+ .map(|(i, c)| match c.to_digit(10) {
+ Some(d) if !every_other || i % 2 == 0 => {
+ char::from_u32(u32::from(zero) + d).unwrap()
+ }
+ _ => c,
+ })
+ .collect()
+ })
+}
+
fn config() -> Criterion {
Criterion::default()
}
diff --git a/native/spark-expr/src/conversion_funcs/string.rs b/native/spark-expr/src/conversion_funcs/string.rs
index f278bafda6..89c18255d0 100644
--- a/native/spark-expr/src/conversion_funcs/string.rs
+++ b/native/spark-expr/src/conversion_funcs/string.rs
@@ -461,37 +461,110 @@ fn cast_string_to_decimal256_impl(
))
}
-/// Normalize fullwidth Unicode digits (U+FF10–U+FF19) to their ASCII equivalents.
+/// Code point of the zero of each run of ten non-ASCII decimal digits (Unicode general
+/// category `Nd`) in the Basic Multilingual Plane, in ascending order. Every run is contiguous,
+/// so the digit at `zero + n` has value `n`.
///
-/// Spark's UTF8String parser treats fullwidth digits as numerically equivalent to
-/// ASCII digits, e.g. "１２３.４５" parses as 123.45. Each fullwidth digit encodes
-/// to exactly three UTF-8 bytes: [0xEF, 0xBC, 0x90+n] for digit n. The ASCII
-/// equivalent is 0x30+n, so the conversion is: third_byte - 0x60.
+/// This is the set of `char`s for which Java's `Character.isDigit(char)` is true, minus ASCII.
+/// It is identical under JDK 17 (Unicode 13) and JDK 21 (Unicode 15); all `Nd` digits added
+/// since lie outside the BMP.
+const BMP_NON_ASCII_DIGIT_ZEROS: [u16; 36] = [
+ 0x0660, // Arabic-Indic
+ 0x06F0, // Extended Arabic-Indic
+ 0x07C0, // NKo
+ 0x0966, // Devanagari
+ 0x09E6, // Bengali
+ 0x0A66, // Gurmukhi
+ 0x0AE6, // Gujarati
+ 0x0B66, // Oriya
+ 0x0BE6, // Tamil
+ 0x0C66, // Telugu
+ 0x0CE6, // Kannada
+ 0x0D66, // Malayalam
+ 0x0DE6, // Sinhala Lith
+ 0x0E50, // Thai
+ 0x0ED0, // Lao
+ 0x0F20, // Tibetan
+ 0x1040, // Myanmar
+ 0x1090, // Myanmar Shan
+ 0x17E0, // Khmer
+ 0x1810, // Mongolian
+ 0x1946, // Limbu
+ 0x19D0, // New Tai Lue
+ 0x1A80, // Tai Tham Hora
+ 0x1A90, // Tai Tham Tham
+ 0x1B50, // Balinese
+ 0x1BB0, // Sundanese
+ 0x1C40, // Lepcha
+ 0x1C50, // Ol Chiki
+ 0xA620, // Vai
+ 0xA8D0, // Saurashtra
+ 0xA900, // Kayah Li
+ 0xA9D0, // Javanese
+ 0xA9F0, // Myanmar Tai Laing
+ 0xAA50, // Cham
+ 0xABF0, // Meetei Mayek
+ 0xFF10, // Fullwidth
+];
+
+/// The value of `c` if it is a non-ASCII decimal digit that Java's `BigDecimal` accepts.
///
-/// All other bytes (ASCII or other multi-byte sequences) are passed through
-/// unchanged, so the output is valid UTF-8 whenever the input is.
-fn normalize_fullwidth_digits(s: &str) -> String {
- let bytes = s.as_bytes();
+/// `BigDecimal` scans UTF-16 code units and tests each with `Character.isDigit(char)`, so only
+/// BMP digits count: a supplementary digit such as U+1D7CE MATHEMATICAL BOLD DIGIT ZERO is a
+/// surrogate pair, neither half of which is a digit, and the parse fails.
+#[inline]
+fn non_ascii_digit_value(c: char) -> Option<u8> {
+ let cp = u32::from(c);
+ if cp > 0xFFFF {
+ return None;
+ }
+ let idx = BMP_NON_ASCII_DIGIT_ZEROS.partition_point(|&zero| u32::from(zero) <= cp);
+ let zero = u32::from(BMP_NON_ASCII_DIGIT_ZEROS[idx.checked_sub(1)?]);
+ let value = cp - zero;
+ (value < 10).then_some(value as u8)
+}
+
+/// Replace every non-ASCII decimal digit that Java's `BigDecimal` accepts with its ASCII
+/// equivalent, e.g. "١٢٣.٤٥" and "１２３.４５" both become "123.45".
+///
+/// Spark parses decimals with `new java.math.BigDecimal(str)`, which reads each digit through
+/// `Character.digit(c, 10)` in both the mantissa and the exponent, so digits from any script
+/// can be mixed freely ("1٢e३" is 12000). All other characters are kept unchanged and are
+/// rejected later by [`parse_decimal_str`].
+fn normalize_unicode_digits(s: &str) -> String {
+ // Every non-ASCII digit is at least two UTF-8 bytes and becomes one, so the output is
+ // never longer than the input.
let mut out = Vec::with_capacity(s.len());
+ let bytes = s.as_bytes();
let mut i = 0;
while i < bytes.len() {
- if i + 2 < bytes.len()
- && bytes[i] == 0xEF
- && bytes[i + 1] == 0xBC
- && bytes[i + 2] >= 0x90
- && bytes[i + 2] <= 0x99
+ let b = bytes[i];
+ // ASCII (including mixed ASCII/fullwidth input) and fullwidth digits, encoded as
+ // [0xEF, 0xBC, 0x90 + n], are the common cases, so they skip the table lookup.
+ if b.is_ascii() {
+ out.push(b);
+ i += 1;
+ } else if b == 0xEF
+ && bytes.get(i + 1) == Some(&0xBC)
+ && matches!(bytes.get(i + 2), Some(0x90..=0x99))
{
// e.g. 0x91 - 0x60 = 0x31 = b'1'
out.push(bytes[i + 2] - 0x60);
i += 3;
} else {
- out.push(bytes[i]);
- i += 1;
+ // `i` is on a char boundary: every branch above advances by a whole char.
+ let mut rest = s[i..].chars();
+ let c = rest.next().unwrap();
+ let next = bytes.len() - rest.as_str().len();
+ match non_ascii_digit_value(c) {
+ Some(d) => out.push(b'0' + d),
+ None => out.extend_from_slice(&bytes[i..next]),
+ }
+ i = next;
}
}
- // SAFETY: we only replace valid 3-byte UTF-8 sequences [EF BC 9X] with a
- // single ASCII byte; all other bytes are copied unchanged, preserving the
- // UTF-8 invariant of the input.
+ // SAFETY: every branch either copies a whole UTF-8 encoded char unchanged or replaces it
+ // with a single ASCII byte, so the output is valid UTF-8 because the input is.
unsafe { String::from_utf8_unchecked(out) }
}
@@ -585,13 +658,13 @@ fn parse_string_to_decimal(input_str: &str, precision: u8, scale: i8) -> SparkRe
// producing NULL.
let trimmed = trim_java_string(input_str);
- // Normalize fullwidth digits to ASCII. Fast path skips the allocation for
- // pure-ASCII strings, which is the common case.
+ // Normalize non-ASCII digits (Arabic-Indic, Devanagari, fullwidth, ...) to ASCII. Fast path
+ // skips the allocation for pure-ASCII strings, which is the common case.
let normalized;
let trimmed = if trimmed.is_ascii() {
trimmed
} else {
- normalized = normalize_fullwidth_digits(trimmed);
+ normalized = normalize_unicode_digits(trimmed);
normalized.as_str()
};
@@ -2335,6 +2408,89 @@ mod tests {
assert!(parse_string_to_decimal(&over_long, 38, 10).is_err());
}
+ #[test]
+ fn test_non_ascii_digit_value() {
+ for &zero in BMP_NON_ASCII_DIGIT_ZEROS.iter() {
+ for n in 0..10u8 {
+ let c = char::from_u32(u32::from(zero) + u32::from(n)).unwrap();
+ assert_eq!(non_ascii_digit_value(c), Some(n), "{c:?}");
+ }
+ // The code point just past each run is not a digit of that run.
+ let after = char::from_u32(u32::from(zero) + 10).unwrap();
+ assert_eq!(non_ascii_digit_value(after), None, "{after:?}");
+ }
+ for c in [
+ '0',
+ '9',
+ 'a',
+ '.',
+ '\u{065F}', // just before Arabic-Indic zero
+ '\u{00B2}', // SUPERSCRIPT TWO (No)
+ '\u{2163}', // ROMAN NUMERAL FOUR (Nl)
+ '\u{2460}', // CIRCLED DIGIT ONE (No)
+ '\u{066B}', // ARABIC DECIMAL SEPARATOR
+ '\u{FFFF}',
+ '\u{1D7CE}', // MATHEMATICAL BOLD DIGIT ZERO: Nd, but outside the BMP
+ '\u{104A3}', // OSMANYA DIGIT THREE: Nd, but outside the BMP
+ ] {
+ assert_eq!(non_ascii_digit_value(c), None, "{c:?}");
+ }
+ }
+
+ #[test]
+ fn test_parse_string_to_decimal_unicode_digits() {
+ // Expected values come from `new java.math.BigDecimal(s)` on JDK 17, which is what
+ // Spark's Decimal.fromString / fromStringANSI call.
+ let ok = |s: &str, precision: u8, scale: i8| {
+ parse_string_to_decimal(s, precision, scale)
+ .unwrap()
+ .unwrap()
+ };
+ assert_eq!(ok("٣", 10, 2), 300); // Arabic-Indic
+ assert_eq!(ok("۳", 10, 2), 300); // Extended Arabic-Indic
+ assert_eq!(ok("३", 10, 2), 300); // Devanagari
+ assert_eq!(ok("৩", 10, 2), 300); // Bengali
+ assert_eq!(ok("๓", 10, 2), 300); // Thai
+ assert_eq!(ok("߃", 10, 2), 300); // NKo
+ assert_eq!(ok("꧓", 10, 2), 300); // Javanese
+ assert_eq!(ok("１２３.４５", 10, 2), 12345); // Fullwidth
+ assert_eq!(ok("1٣", 10, 2), 1300);
+ assert_eq!(ok("-٣", 10, 2), -300);
+ assert_eq!(ok("+٣.٣٣", 10, 2), 333);
+ assert_eq!(ok(".٣", 10, 2), 30);
+ assert_eq!(ok("١٢٣.٤٥٥", 10, 2), 12346); // HALF_UP rounding
+ assert_eq!(ok("1e٣", 10, 2), 100000); // non-ASCII exponent digits
+ assert_eq!(ok("1E-٣", 10, 3), 1);
+ assert_eq!(ok("1e+٣", 10, 2), 100000);
+ assert_eq!(ok("1٢e३", 10, 0), 12000); // mixed scripts
+ assert_eq!(ok(" \t٣\n", 10, 2), 300); // ASCII trim still applies
+ assert_eq!(ok(&"٣".repeat(20), 38, 0), 33333333333333333333);
+
+ // Rejected by BigDecimal: not Nd, or Nd outside the BMP.
+ for s in [
+ "\u{1D7D0}", // MATHEMATICAL BOLD DIGIT TWO
+ "1\u{1D7D0}",
+ "1e\u{1D7D0}",
+ "\u{2163}", // ROMAN NUMERAL FOUR
+ "\u{00B2}", // SUPERSCRIPT TWO
+ "١\u{066B}٥", // ARABIC DECIMAL SEPARATOR is not '.'
+ "\u{00A0}٣", // non-ASCII whitespace is not trimmed
+ ] {
+ // The error reports the original string, not the normalized one.
+ match parse_string_to_decimal(s, 10, 2) {
+ Err(SparkError::CastInvalidValue { value, .. }) => assert_eq!(value, s),
+ other => panic!("{s:?}: {other:?}"),
+ }
+ }
+
+ // Overflow is reported as such, not as an invalid input.
+ let err = parse_string_to_decimal("٣٣٣٣٣٣٣٣٣", 10, 2).unwrap_err();
+ assert!(
+ matches!(err, SparkError::NumericValueOutOfRange { .. }),
+ "{err}"
+ );
+ }
+
#[test]
#[cfg_attr(miri, ignore)] // test takes too long with miri
fn test_cast_string_to_timestamp() {
diff --git a/spark/src/test/resources/sql-tests/expressions/cast/cast_string_to_decimal_unicode_digits.sql b/spark/src/test/resources/sql-tests/expressions/cast/cast_string_to_decimal_unicode_digits.sql
new file mode 100644
index 0000000000..f8d22a1d55
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/cast/cast_string_to_decimal_unicode_digits.sql
@@ -0,0 +1,88 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- cast(string as decimal) with non-ASCII digits, non-ANSI mode. Spark parses the trimmed string
+-- with `new java.math.BigDecimal(s)`, which accepts every char for which
+-- `Character.isDigit(char)` is true: all Unicode `Nd` digits in the Basic Multilingual Plane,
+-- in the mantissa and in the exponent, mixed freely across scripts. Supplementary-plane digits
+-- (e.g. U+1D7D0 MATHEMATICAL BOLD DIGIT TWO) are surrogate pairs and are rejected.
+--
+-- The integral, floating-point and date casts of the same strings are ASCII-only in Spark
+-- (UTF8String.toLong, Double.parseDouble, SparkDateTimeUtils.stringToDate) and must stay NULL.
+
+-- Config: spark.sql.ansi.enabled=false
+
+statement
+CREATE TABLE test_cast_str_dec_unicode(id int, s string) USING parquet
+
+-- 1: U+0663 ARABIC-INDIC DIGIT THREE
+-- 2: U+06F3 EXTENDED ARABIC-INDIC DIGIT THREE
+-- 3: U+0969 DEVANAGARI DIGIT THREE
+-- 4: U+09E9 BENGALI DIGIT THREE
+-- 5: U+0E53 THAI DIGIT THREE
+-- 6: U+07C3 NKO DIGIT THREE
+-- 7: U+A9D3 JAVANESE DIGIT THREE
+-- 8: fullwidth digits
+-- 9-17: signs, '.', HALF_UP rounding, exponent digits, mixed scripts, ASCII trim
+-- 18: 20 Arabic-Indic digits, too many for the cast targets below
+-- 19-25: rejected by BigDecimal: supplementary-plane Nd digits, ROMAN NUMERAL FOUR (Nl),
+-- SUPERSCRIPT TWO (No), ARABIC DECIMAL SEPARATOR, leading NBSP (not trimmed)
+statement
+INSERT INTO test_cast_str_dec_unicode VALUES
+ (1, '٣'),
+ (2, '۳'),
+ (3, '३'),
+ (4, '৩'),
+ (5, '๓'),
+ (6, '߃'),
+ (7, '꧓'),
+ (8, '１２３.４５'),
+ (9, '1٣'),
+ (10, '-٣'),
+ (11, '+٣.٣٣'),
+ (12, '.٣'),
+ (13, '١٢٣.٤٥٥'),
+ (14, '1e٣'),
+ (15, '1E-٣'),
+ (16, '1٢e३'),
+ (17, concat(' ٣', chr(9))),
+ (18, '٣٣٣٣٣٣٣٣٣٣٣٣٣٣٣٣٣٣٣٣'),
+ (19, '𝟐'),
+ (20, '1𝟐'),
+ (21, '1e𝟐'),
+ (22, 'Ⅳ'),
+ (23, '²'),
+ (24, '١٫٥'),
+ (25, concat(chr(160), '٣')),
+ (26, NULL)
+
+query
+SELECT id, cast(s as decimal(10,2)), cast(s as decimal(38,10)), cast(s as decimal(5,0))
+FROM test_cast_str_dec_unicode ORDER BY id
+
+query
+SELECT id, try_cast(s as decimal(10,2)), try_cast(s as decimal(38,0))
+FROM test_cast_str_dec_unicode ORDER BY id
+
+-- Spark's integral, floating-point and date parsers only accept ASCII digits
+query
+SELECT id, cast(s as int), cast(s as bigint), cast(s as double), cast(s as date)
+FROM test_cast_str_dec_unicode ORDER BY id
+
+-- literal arguments
+query
+SELECT cast('٣' as decimal(10,2)), cast('१२३.४५' as decimal(10,2)), cast('𝟐' as decimal(10,2))
diff --git a/spark/src/test/resources/sql-tests/expressions/cast/cast_string_to_decimal_unicode_digits_ansi.sql b/spark/src/test/resources/sql-tests/expressions/cast/cast_string_to_decimal_unicode_digits_ansi.sql
new file mode 100644
index 0000000000..f546893dfa
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/cast/cast_string_to_decimal_unicode_digits_ansi.sql
@@ -0,0 +1,76 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- cast(string as decimal) with non-ASCII digits, ANSI mode. Spark parses the trimmed string
+-- with `new java.math.BigDecimal(s)`, which accepts every Unicode `Nd` digit in the Basic
+-- Multilingual Plane, so these must cast to a value rather than raise CAST_INVALID_INPUT.
+-- Supplementary-plane digits are rejected by BigDecimal and must still raise.
+
+-- Config: spark.sql.ansi.enabled=true
+
+statement
+CREATE TABLE test_cast_str_dec_unicode_ansi(id int, s string) USING parquet
+
+-- Arabic-Indic, Devanagari, Thai, fullwidth, signs, '.', rounding, exponent digits, mixed
+-- scripts and ASCII trim; every value fits decimal(10,2)
+statement
+INSERT INTO test_cast_str_dec_unicode_ansi VALUES
+ (1, '٣'),
+ (2, '३'),
+ (3, '๓'),
+ (4, '１２３.４５'),
+ (5, '-٣'),
+ (6, '+٣.٣٣'),
+ (7, '١٢٣.٤٥٥'),
+ (8, '1e٣'),
+ (9, '1E-٣'),
+ (10, '1٢e३'),
+ (11, concat(' ٣', chr(9))),
+ (12, NULL)
+
+statement
+CREATE TABLE test_cast_str_dec_unicode_ansi_bad(s string) USING parquet
+
+-- U+1D7D0 MATHEMATICAL BOLD DIGIT TWO is Nd but outside the BMP
+statement
+INSERT INTO test_cast_str_dec_unicode_ansi_bad VALUES ('𝟐')
+
+statement
+CREATE TABLE test_cast_str_dec_unicode_ansi_big(s string) USING parquet
+
+statement
+INSERT INTO test_cast_str_dec_unicode_ansi_big VALUES ('٣٣٣٣٣٣٣٣٣')
+
+query
+SELECT id, cast(s as decimal(10,2)), cast(s as decimal(38,10))
+FROM test_cast_str_dec_unicode_ansi ORDER BY id
+
+query
+SELECT id, try_cast(s as decimal(10,2)) FROM test_cast_str_dec_unicode_ansi ORDER BY id
+
+query expect_error(CAST_INVALID_INPUT)
+SELECT cast(s as decimal(10,2)) FROM test_cast_str_dec_unicode_ansi_bad
+
+-- the value is valid but does not fit decimal(10,2)
+query expect_error(NUMERIC_VALUE_OUT_OF_RANGE)
+SELECT cast(s as decimal(10,2)) FROM test_cast_str_dec_unicode_ansi_big
+
+-- try_cast suppresses both errors
+query
+SELECT try_cast(s as decimal(10,2)) FROM test_cast_str_dec_unicode_ansi_bad
+UNION ALL
+SELECT try_cast(s as decimal(10,2)) FROM test_cast_str_dec_unicode_ansi_big
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org