You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2024/01/09 20:35:43 UTC

(arrow-datafusion) branch main updated: Change `ScalarValue::{List, LargeList, FixedSizedList}` to take specific types rather than `ArrayRef` (#8562)

This is an automated email from the ASF dual-hosted git repository.

alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git


The following commit(s) were added to refs/heads/main by this push:
     new f8d860365e Change `ScalarValue::{List, LargeList, FixedSizedList}` to take specific types rather than `ArrayRef` (#8562)
f8d860365e is described below

commit f8d860365ef17d05cb033c1f39928cac6620e2f2
Author: Randall Spears <ra...@rspears.me>
AuthorDate: Tue Jan 9 13:35:38 2024 -0700

    Change `ScalarValue::{List, LargeList, FixedSizedList}` to take specific types rather than `ArrayRef` (#8562)
    
    * Change ScalarValue::List type signature
    
    Also ScalarValue::LargeList and ScalarValue::FixedSizeList
    
    * Formatting/cleanup
    
    * Remove duplicate match statements
    
    * Add back scalar eq_array test for List
    
    * Formatting
    
    * Reduce code duplication
    
    * Fix merge conflict
    
    * Fix post-merge compile errors
    
    * Remove redundant partial_cmp implementation
    
    * improve
    
    * Cargo fmt fix
    
    * Reduce duplication in formatter
    
    * Reduce more duplication
    
    * Fix test error
    
    ---------
    
    Co-authored-by: Spears Randall <Sp...@JohnDeere.com>
    Co-authored-by: Andrew Lamb <an...@nerdnetworks.org>
---
 datafusion/common/src/scalar.rs                    | 305 ++++++++++++---------
 .../src/simplify_expressions/expr_simplifier.rs    |   4 +-
 .../src/aggregate/array_agg_distinct.rs            |   6 +-
 .../physical-expr/src/aggregate/count_distinct.rs  |   4 +-
 datafusion/physical-expr/src/aggregate/tdigest.rs  |   6 +-
 datafusion/proto/src/logical_plan/from_proto.rs    |  13 +-
 datafusion/proto/src/logical_plan/to_proto.rs      | 100 +++----
 7 files changed, 243 insertions(+), 195 deletions(-)

diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs
index 48878aa9bd..8820ca9942 100644
--- a/datafusion/common/src/scalar.rs
+++ b/datafusion/common/src/scalar.rs
@@ -21,6 +21,7 @@ use std::borrow::Borrow;
 use std::cmp::Ordering;
 use std::collections::HashSet;
 use std::convert::{Infallible, TryInto};
+use std::hash::Hash;
 use std::str::FromStr;
 use std::{convert::TryFrom, fmt, iter::repeat, sync::Arc};
 
@@ -142,13 +143,13 @@ pub enum ScalarValue {
     /// Fixed size list scalar.
     ///
     /// The array must be a FixedSizeListArray with length 1.
-    FixedSizeList(ArrayRef),
+    FixedSizeList(Arc<FixedSizeListArray>),
     /// Represents a single element of a [`ListArray`] as an [`ArrayRef`]
     ///
     /// The array must be a ListArray with length 1.
-    List(ArrayRef),
+    List(Arc<ListArray>),
     /// The array must be a LargeListArray with length 1.
-    LargeList(ArrayRef),
+    LargeList(Arc<LargeListArray>),
     /// Date stored as a signed 32bit int days since UNIX epoch 1970-01-01
     Date32(Option<i32>),
     /// Date stored as a signed 64bit int milliseconds since UNIX epoch 1970-01-01
@@ -360,45 +361,13 @@ impl PartialOrd for ScalarValue {
             (FixedSizeBinary(_, _), _) => None,
             (LargeBinary(v1), LargeBinary(v2)) => v1.partial_cmp(v2),
             (LargeBinary(_), _) => None,
-            (List(arr1), List(arr2))
-            | (FixedSizeList(arr1), FixedSizeList(arr2))
-            | (LargeList(arr1), LargeList(arr2)) => {
-                // ScalarValue::List / ScalarValue::FixedSizeList / ScalarValue::LargeList are ensure to have length 1
-                assert_eq!(arr1.len(), 1);
-                assert_eq!(arr2.len(), 1);
-
-                if arr1.data_type() != arr2.data_type() {
-                    return None;
-                }
-
-                fn first_array_for_list(arr: &ArrayRef) -> ArrayRef {
-                    if let Some(arr) = arr.as_list_opt::<i32>() {
-                        arr.value(0)
-                    } else if let Some(arr) = arr.as_list_opt::<i64>() {
-                        arr.value(0)
-                    } else if let Some(arr) = arr.as_fixed_size_list_opt() {
-                        arr.value(0)
-                    } else {
-                        unreachable!("Since only List / LargeList / FixedSizeList are supported, this should never happen")
-                    }
-                }
-
-                let arr1 = first_array_for_list(arr1);
-                let arr2 = first_array_for_list(arr2);
-
-                let lt_res = arrow::compute::kernels::cmp::lt(&arr1, &arr2).ok()?;
-                let eq_res = arrow::compute::kernels::cmp::eq(&arr1, &arr2).ok()?;
-
-                for j in 0..lt_res.len() {
-                    if lt_res.is_valid(j) && lt_res.value(j) {
-                        return Some(Ordering::Less);
-                    }
-                    if eq_res.is_valid(j) && !eq_res.value(j) {
-                        return Some(Ordering::Greater);
-                    }
-                }
-
-                Some(Ordering::Equal)
+            // ScalarValue::List / ScalarValue::FixedSizeList / ScalarValue::LargeList are ensure to have length 1
+            (List(arr1), List(arr2)) => partial_cmp_list(arr1.as_ref(), arr2.as_ref()),
+            (FixedSizeList(arr1), FixedSizeList(arr2)) => {
+                partial_cmp_list(arr1.as_ref(), arr2.as_ref())
+            }
+            (LargeList(arr1), LargeList(arr2)) => {
+                partial_cmp_list(arr1.as_ref(), arr2.as_ref())
             }
             (List(_), _) | (LargeList(_), _) | (FixedSizeList(_), _) => None,
             (Date32(v1), Date32(v2)) => v1.partial_cmp(v2),
@@ -464,6 +433,44 @@ impl PartialOrd for ScalarValue {
     }
 }
 
+/// List/LargeList/FixedSizeList scalars always have a single element
+/// array. This function returns that array
+fn first_array_for_list(arr: &dyn Array) -> ArrayRef {
+    assert_eq!(arr.len(), 1);
+    if let Some(arr) = arr.as_list_opt::<i32>() {
+        arr.value(0)
+    } else if let Some(arr) = arr.as_list_opt::<i64>() {
+        arr.value(0)
+    } else if let Some(arr) = arr.as_fixed_size_list_opt() {
+        arr.value(0)
+    } else {
+        unreachable!("Since only List / LargeList / FixedSizeList are supported, this should never happen")
+    }
+}
+
+/// Compares two List/LargeList/FixedSizeList scalars
+fn partial_cmp_list(arr1: &dyn Array, arr2: &dyn Array) -> Option<Ordering> {
+    if arr1.data_type() != arr2.data_type() {
+        return None;
+    }
+    let arr1 = first_array_for_list(arr1);
+    let arr2 = first_array_for_list(arr2);
+
+    let lt_res = arrow::compute::kernels::cmp::lt(&arr1, &arr2).ok()?;
+    let eq_res = arrow::compute::kernels::cmp::eq(&arr1, &arr2).ok()?;
+
+    for j in 0..lt_res.len() {
+        if lt_res.is_valid(j) && lt_res.value(j) {
+            return Some(Ordering::Less);
+        }
+        if eq_res.is_valid(j) && !eq_res.value(j) {
+            return Some(Ordering::Greater);
+        }
+    }
+
+    Some(Ordering::Equal)
+}
+
 impl Eq for ScalarValue {}
 
 //Float wrapper over f32/f64. Just because we cannot build std::hash::Hash for floats directly we have to do it through type wrapper
@@ -517,14 +524,14 @@ impl std::hash::Hash for ScalarValue {
             Binary(v) => v.hash(state),
             FixedSizeBinary(_, v) => v.hash(state),
             LargeBinary(v) => v.hash(state),
-            List(arr) | LargeList(arr) | FixedSizeList(arr) => {
-                let arrays = vec![arr.to_owned()];
-                let hashes_buffer = &mut vec![0; arr.len()];
-                let random_state = ahash::RandomState::with_seeds(0, 0, 0, 0);
-                let hashes =
-                    create_hashes(&arrays, &random_state, hashes_buffer).unwrap();
-                // Hash back to std::hash::Hasher
-                hashes.hash(state);
+            List(arr) => {
+                hash_list(arr.to_owned() as ArrayRef, state);
+            }
+            LargeList(arr) => {
+                hash_list(arr.to_owned() as ArrayRef, state);
+            }
+            FixedSizeList(arr) => {
+                hash_list(arr.to_owned() as ArrayRef, state);
             }
             Date32(v) => v.hash(state),
             Date64(v) => v.hash(state),
@@ -557,6 +564,15 @@ impl std::hash::Hash for ScalarValue {
     }
 }
 
+fn hash_list<H: std::hash::Hasher>(arr: ArrayRef, state: &mut H) {
+    let arrays = vec![arr.to_owned()];
+    let hashes_buffer = &mut vec![0; arr.len()];
+    let random_state = ahash::RandomState::with_seeds(0, 0, 0, 0);
+    let hashes = create_hashes(&arrays, &random_state, hashes_buffer).unwrap();
+    // Hash back to std::hash::Hasher
+    hashes.hash(state);
+}
+
 /// Return a reference to the values array and the index into it for a
 /// dictionary array
 ///
@@ -942,9 +958,9 @@ impl ScalarValue {
             ScalarValue::Binary(_) => DataType::Binary,
             ScalarValue::FixedSizeBinary(sz, _) => DataType::FixedSizeBinary(*sz),
             ScalarValue::LargeBinary(_) => DataType::LargeBinary,
-            ScalarValue::List(arr)
-            | ScalarValue::LargeList(arr)
-            | ScalarValue::FixedSizeList(arr) => arr.data_type().to_owned(),
+            ScalarValue::List(arr) => arr.data_type().to_owned(),
+            ScalarValue::LargeList(arr) => arr.data_type().to_owned(),
+            ScalarValue::FixedSizeList(arr) => arr.data_type().to_owned(),
             ScalarValue::Date32(_) => DataType::Date32,
             ScalarValue::Date64(_) => DataType::Date64,
             ScalarValue::Time32Second(_) => DataType::Time32(TimeUnit::Second),
@@ -1147,9 +1163,9 @@ impl ScalarValue {
             ScalarValue::LargeBinary(v) => v.is_none(),
             // arr.len() should be 1 for a list scalar, but we don't seem to
             // enforce that anywhere, so we still check against array length.
-            ScalarValue::List(arr)
-            | ScalarValue::LargeList(arr)
-            | ScalarValue::FixedSizeList(arr) => arr.len() == arr.null_count(),
+            ScalarValue::List(arr) => arr.len() == arr.null_count(),
+            ScalarValue::LargeList(arr) => arr.len() == arr.null_count(),
+            ScalarValue::FixedSizeList(arr) => arr.len() == arr.null_count(),
             ScalarValue::Date32(v) => v.is_none(),
             ScalarValue::Date64(v) => v.is_none(),
             ScalarValue::Time32Second(v) => v.is_none(),
@@ -1695,17 +1711,16 @@ impl ScalarValue {
     ///    ScalarValue::Int32(Some(2))
     /// ];
     ///
-    /// let array = ScalarValue::new_list(&scalars, &DataType::Int32);
-    /// let result = as_list_array(&array).unwrap();
+    /// let result = ScalarValue::new_list(&scalars, &DataType::Int32);
     ///
     /// let expected = ListArray::from_iter_primitive::<Int32Type, _, _>(
     ///     vec![
     ///        Some(vec![Some(1), None, Some(2)])
     ///     ]);
     ///
-    /// assert_eq!(result, &expected);
+    /// assert_eq!(*result, expected);
     /// ```
-    pub fn new_list(values: &[ScalarValue], data_type: &DataType) -> ArrayRef {
+    pub fn new_list(values: &[ScalarValue], data_type: &DataType) -> Arc<ListArray> {
         let values = if values.is_empty() {
             new_empty_array(data_type)
         } else {
@@ -1730,17 +1745,19 @@ impl ScalarValue {
     ///    ScalarValue::Int32(Some(2))
     /// ];
     ///
-    /// let array = ScalarValue::new_large_list(&scalars, &DataType::Int32);
-    /// let result = as_large_list_array(&array).unwrap();
+    /// let result = ScalarValue::new_large_list(&scalars, &DataType::Int32);
     ///
     /// let expected = LargeListArray::from_iter_primitive::<Int32Type, _, _>(
     ///     vec![
     ///        Some(vec![Some(1), None, Some(2)])
     ///     ]);
     ///
-    /// assert_eq!(result, &expected);
+    /// assert_eq!(*result, expected);
     /// ```
-    pub fn new_large_list(values: &[ScalarValue], data_type: &DataType) -> ArrayRef {
+    pub fn new_large_list(
+        values: &[ScalarValue],
+        data_type: &DataType,
+    ) -> Arc<LargeListArray> {
         let values = if values.is_empty() {
             new_empty_array(data_type)
         } else {
@@ -1876,14 +1893,14 @@ impl ScalarValue {
                         .collect::<LargeBinaryArray>(),
                 ),
             },
-            ScalarValue::List(arr)
-            | ScalarValue::LargeList(arr)
-            | ScalarValue::FixedSizeList(arr) => {
-                let arrays = std::iter::repeat(arr.as_ref())
-                    .take(size)
-                    .collect::<Vec<_>>();
-                arrow::compute::concat(arrays.as_slice())
-                    .map_err(|e| arrow_datafusion_err!(e))?
+            ScalarValue::List(arr) => {
+                Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?
+            }
+            ScalarValue::LargeList(arr) => {
+                Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?
+            }
+            ScalarValue::FixedSizeList(arr) => {
+                Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?
             }
             ScalarValue::Date32(e) => {
                 build_array_from_option!(Date32, Date32Array, e, size)
@@ -2040,6 +2057,11 @@ impl ScalarValue {
         }
     }
 
+    fn list_to_array_of_size(arr: &dyn Array, size: usize) -> Result<ArrayRef> {
+        let arrays = std::iter::repeat(arr).take(size).collect::<Vec<_>>();
+        arrow::compute::concat(arrays.as_slice()).map_err(|e| arrow_datafusion_err!(e))
+    }
+
     /// Retrieve ScalarValue for each row in `array`
     ///
     /// Example
@@ -2433,11 +2455,14 @@ impl ScalarValue {
             ScalarValue::LargeBinary(val) => {
                 eq_array_primitive!(array, index, LargeBinaryArray, val)?
             }
-            ScalarValue::List(arr)
-            | ScalarValue::LargeList(arr)
-            | ScalarValue::FixedSizeList(arr) => {
-                let right = array.slice(index, 1);
-                arr == &right
+            ScalarValue::List(arr) => {
+                Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)
+            }
+            ScalarValue::LargeList(arr) => {
+                Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)
+            }
+            ScalarValue::FixedSizeList(arr) => {
+                Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)
             }
             ScalarValue::Date32(val) => {
                 eq_array_primitive!(array, index, Date32Array, val)?
@@ -2515,6 +2540,11 @@ impl ScalarValue {
         })
     }
 
+    fn eq_array_list(arr1: &ArrayRef, arr2: &ArrayRef, index: usize) -> bool {
+        let right = arr2.slice(index, 1);
+        arr1 == &right
+    }
+
     /// Estimate size if bytes including `Self`. For values with internal containers such as `String`
     /// includes the allocated size (`capacity`) rather than the current length (`len`)
     pub fn size(&self) -> usize {
@@ -2561,9 +2591,9 @@ impl ScalarValue {
                 | ScalarValue::LargeBinary(b) => {
                     b.as_ref().map(|b| b.capacity()).unwrap_or_default()
                 }
-                ScalarValue::List(arr)
-                | ScalarValue::LargeList(arr)
-                | ScalarValue::FixedSizeList(arr) => arr.get_array_memory_size(),
+                ScalarValue::List(arr) => arr.get_array_memory_size(),
+                ScalarValue::LargeList(arr) => arr.get_array_memory_size(),
+                ScalarValue::FixedSizeList(arr) => arr.get_array_memory_size(),
                 ScalarValue::Struct(vals, fields) => {
                     vals.as_ref()
                         .map(|vals| {
@@ -2865,14 +2895,19 @@ impl TryFrom<&DataType> for ScalarValue {
                 Box::new(value_type.as_ref().try_into()?),
             ),
             // `ScalaValue::List` contains single element `ListArray`.
-            DataType::List(field) => ScalarValue::List(new_null_array(
-                &DataType::List(Arc::new(Field::new(
-                    "item",
-                    field.data_type().clone(),
-                    true,
-                ))),
-                1,
-            )),
+            DataType::List(field) => ScalarValue::List(
+                new_null_array(
+                    &DataType::List(Arc::new(Field::new(
+                        "item",
+                        field.data_type().clone(),
+                        true,
+                    ))),
+                    1,
+                )
+                .as_list::<i32>()
+                .to_owned()
+                .into(),
+            ),
             DataType::Struct(fields) => ScalarValue::Struct(None, fields.clone()),
             DataType::Null => ScalarValue::Null,
             _ => {
@@ -2937,16 +2972,9 @@ impl fmt::Display for ScalarValue {
                 )?,
                 None => write!(f, "NULL")?,
             },
-            ScalarValue::List(arr)
-            | ScalarValue::LargeList(arr)
-            | ScalarValue::FixedSizeList(arr) => {
-                // ScalarValue List should always have a single element
-                assert_eq!(arr.len(), 1);
-                let options = FormatOptions::default().with_display_error(true);
-                let formatter = ArrayFormatter::try_new(arr, &options).unwrap();
-                let value_formatter = formatter.value(0);
-                write!(f, "{value_formatter}")?
-            }
+            ScalarValue::List(arr) => fmt_list(arr.to_owned() as ArrayRef, f)?,
+            ScalarValue::LargeList(arr) => fmt_list(arr.to_owned() as ArrayRef, f)?,
+            ScalarValue::FixedSizeList(arr) => fmt_list(arr.to_owned() as ArrayRef, f)?,
             ScalarValue::Date32(e) => format_option!(f, e)?,
             ScalarValue::Date64(e) => format_option!(f, e)?,
             ScalarValue::Time32Second(e) => format_option!(f, e)?,
@@ -2979,6 +3007,16 @@ impl fmt::Display for ScalarValue {
     }
 }
 
+fn fmt_list(arr: ArrayRef, f: &mut fmt::Formatter) -> fmt::Result {
+    // ScalarValue List, LargeList, FixedSizeList should always have a single element
+    assert_eq!(arr.len(), 1);
+    let options = FormatOptions::default().with_display_error(true);
+    let formatter =
+        ArrayFormatter::try_new(arr.as_ref() as &dyn Array, &options).unwrap();
+    let value_formatter = formatter.value(0);
+    write!(f, "{value_formatter}")
+}
+
 impl fmt::Debug for ScalarValue {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
@@ -3182,15 +3220,14 @@ mod tests {
             ScalarValue::from("data-fusion"),
         ];
 
-        let array = ScalarValue::new_list(scalars.as_slice(), &DataType::Utf8);
+        let result = ScalarValue::new_list(scalars.as_slice(), &DataType::Utf8);
 
         let expected = array_into_list_array(Arc::new(StringArray::from(vec![
             "rust",
             "arrow",
             "data-fusion",
         ])));
-        let result = as_list_array(&array);
-        assert_eq!(result, &expected);
+        assert_eq!(*result, expected);
     }
 
     fn build_list<O: OffsetSizeTrait>(
@@ -3226,9 +3263,9 @@ mod tests {
                 };
 
                 if O::IS_LARGE {
-                    ScalarValue::LargeList(arr)
+                    ScalarValue::LargeList(arr.as_list::<i64>().to_owned().into())
                 } else {
-                    ScalarValue::List(arr)
+                    ScalarValue::List(arr.as_list::<i32>().to_owned().into())
                 }
             })
             .collect()
@@ -3311,18 +3348,16 @@ mod tests {
             ]));
 
         let fsl_array: ArrayRef =
-            Arc::new(FixedSizeListArray::from_iter_primitive::<Int32Type, _, _>(
-                vec![
-                    Some(vec![Some(0), Some(1), Some(2)]),
-                    None,
-                    Some(vec![Some(3), None, Some(5)]),
-                ],
-                3,
-            ));
+            Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+                Some(vec![Some(0), Some(1), Some(2)]),
+                None,
+                Some(vec![Some(3), None, Some(5)]),
+            ]));
 
         for arr in [list_array, fsl_array] {
             for i in 0..arr.len() {
-                let scalar = ScalarValue::List(arr.slice(i, 1));
+                let scalar =
+                    ScalarValue::List(arr.slice(i, 1).as_list::<i32>().to_owned().into());
                 assert!(scalar.eq_array(&arr, i).unwrap());
             }
         }
@@ -3676,8 +3711,7 @@ mod tests {
 
     #[test]
     fn scalar_list_null_to_array() {
-        let list_array_ref = ScalarValue::new_list(&[], &DataType::UInt64);
-        let list_array = as_list_array(&list_array_ref);
+        let list_array = ScalarValue::new_list(&[], &DataType::UInt64);
 
         assert_eq!(list_array.len(), 1);
         assert_eq!(list_array.values().len(), 0);
@@ -3685,8 +3719,7 @@ mod tests {
 
     #[test]
     fn scalar_large_list_null_to_array() {
-        let list_array_ref = ScalarValue::new_large_list(&[], &DataType::UInt64);
-        let list_array = as_large_list_array(&list_array_ref);
+        let list_array = ScalarValue::new_large_list(&[], &DataType::UInt64);
 
         assert_eq!(list_array.len(), 1);
         assert_eq!(list_array.values().len(), 0);
@@ -3699,8 +3732,7 @@ mod tests {
             ScalarValue::UInt64(None),
             ScalarValue::UInt64(Some(101)),
         ];
-        let list_array_ref = ScalarValue::new_list(&values, &DataType::UInt64);
-        let list_array = as_list_array(&list_array_ref);
+        let list_array = ScalarValue::new_list(&values, &DataType::UInt64);
         assert_eq!(list_array.len(), 1);
         assert_eq!(list_array.values().len(), 3);
 
@@ -3720,8 +3752,7 @@ mod tests {
             ScalarValue::UInt64(None),
             ScalarValue::UInt64(Some(101)),
         ];
-        let list_array_ref = ScalarValue::new_large_list(&values, &DataType::UInt64);
-        let list_array = as_large_list_array(&list_array_ref);
+        let list_array = ScalarValue::new_large_list(&values, &DataType::UInt64);
         assert_eq!(list_array.len(), 1);
         assert_eq!(list_array.values().len(), 3);
 
@@ -3959,10 +3990,15 @@ mod tests {
         let data_type = &data_type;
         let scalar: ScalarValue = data_type.try_into().unwrap();
 
-        let expected = ScalarValue::List(new_null_array(
-            &DataType::List(Arc::new(Field::new("item", DataType::Int32, true))),
-            1,
-        ));
+        let expected = ScalarValue::List(
+            new_null_array(
+                &DataType::List(Arc::new(Field::new("item", DataType::Int32, true))),
+                1,
+            )
+            .as_list::<i32>()
+            .to_owned()
+            .into(),
+        );
 
         assert_eq!(expected, scalar)
     }
@@ -3977,14 +4013,19 @@ mod tests {
         let data_type = &data_type;
         let scalar: ScalarValue = data_type.try_into().unwrap();
 
-        let expected = ScalarValue::List(new_null_array(
-            &DataType::List(Arc::new(Field::new(
-                "item",
-                DataType::List(Arc::new(Field::new("item", DataType::Int32, true))),
-                true,
-            ))),
-            1,
-        ));
+        let expected = ScalarValue::List(
+            new_null_array(
+                &DataType::List(Arc::new(Field::new(
+                    "item",
+                    DataType::List(Arc::new(Field::new("item", DataType::Int32, true))),
+                    true,
+                ))),
+                1,
+            )
+            .as_list::<i32>()
+            .to_owned()
+            .into(),
+        );
 
         assert_eq!(expected, scalar)
     }
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
index 7d09aec7e7..dbdfb856a7 100644
--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
@@ -27,7 +27,7 @@ use crate::simplify_expressions::regex::simplify_regex_expr;
 use crate::simplify_expressions::SimplifyInfo;
 
 use arrow::{
-    array::new_null_array,
+    array::{new_null_array, AsArray},
     datatypes::{DataType, Field, Schema},
     record_batch::RecordBatch,
 };
@@ -396,7 +396,7 @@ impl<'a> ConstEvaluator<'a> {
                         a.len()
                     )
                 } else if as_list_array(&a).is_ok() || as_large_list_array(&a).is_ok() {
-                    Ok(ScalarValue::List(a))
+                    Ok(ScalarValue::List(a.as_list().to_owned().into()))
                 } else {
                     // Non-ListArray
                     ScalarValue::try_from_array(&a, 0)
diff --git a/datafusion/physical-expr/src/aggregate/array_agg_distinct.rs b/datafusion/physical-expr/src/aggregate/array_agg_distinct.rs
index 1efae424cc..2d263a42e0 100644
--- a/datafusion/physical-expr/src/aggregate/array_agg_distinct.rs
+++ b/datafusion/physical-expr/src/aggregate/array_agg_distinct.rs
@@ -186,7 +186,6 @@ mod tests {
     use arrow::array::{ArrayRef, Int32Array};
     use arrow::datatypes::{DataType, Schema};
     use arrow::record_batch::RecordBatch;
-    use arrow_array::cast::as_list_array;
     use arrow_array::types::Int32Type;
     use arrow_array::{Array, ListArray};
     use arrow_buffer::OffsetBuffer;
@@ -196,10 +195,7 @@ mod tests {
     // arrow::compute::sort cann't sort ListArray directly, so we need to sort the inner primitive array and wrap it back into ListArray.
     fn sort_list_inner(arr: ScalarValue) -> ScalarValue {
         let arr = match arr {
-            ScalarValue::List(arr) => {
-                let list_arr = as_list_array(&arr);
-                list_arr.value(0)
-            }
+            ScalarValue::List(arr) => arr.value(0),
             _ => {
                 panic!("Expected ScalarValue::List, got {:?}", arr)
             }
diff --git a/datafusion/physical-expr/src/aggregate/count_distinct.rs b/datafusion/physical-expr/src/aggregate/count_distinct.rs
index f7c13948b2..021c33fb94 100644
--- a/datafusion/physical-expr/src/aggregate/count_distinct.rs
+++ b/datafusion/physical-expr/src/aggregate/count_distinct.rs
@@ -292,7 +292,7 @@ where
         let arr = Arc::new(PrimitiveArray::<T>::from_iter_values(
             self.values.iter().cloned(),
         )) as ArrayRef;
-        let list = Arc::new(array_into_list_array(arr)) as ArrayRef;
+        let list = Arc::new(array_into_list_array(arr));
         Ok(vec![ScalarValue::List(list)])
     }
 
@@ -378,7 +378,7 @@ where
         let arr = Arc::new(PrimitiveArray::<T>::from_iter_values(
             self.values.iter().map(|v| v.0),
         )) as ArrayRef;
-        let list = Arc::new(array_into_list_array(arr)) as ArrayRef;
+        let list = Arc::new(array_into_list_array(arr));
         Ok(vec![ScalarValue::List(list)])
     }
 
diff --git a/datafusion/physical-expr/src/aggregate/tdigest.rs b/datafusion/physical-expr/src/aggregate/tdigest.rs
index 90f5244f47..78708df94c 100644
--- a/datafusion/physical-expr/src/aggregate/tdigest.rs
+++ b/datafusion/physical-expr/src/aggregate/tdigest.rs
@@ -28,7 +28,6 @@
 //! [Facebook's Folly TDigest]: https://github.com/facebook/folly/blob/main/folly/stats/TDigest.h
 
 use arrow::datatypes::DataType;
-use arrow_array::cast::as_list_array;
 use arrow_array::types::Float64Type;
 use datafusion_common::cast::as_primitive_array;
 use datafusion_common::Result;
@@ -606,11 +605,10 @@ impl TDigest {
 
         let centroids: Vec<_> = match &state[5] {
             ScalarValue::List(arr) => {
-                let list_array = as_list_array(arr);
-                let arr = list_array.values();
+                let array = arr.values();
 
                 let f64arr =
-                    as_primitive_array::<Float64Type>(arr).expect("expected f64 array");
+                    as_primitive_array::<Float64Type>(array).expect("expected f64 array");
                 f64arr
                     .values()
                     .chunks(2)
diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rs
index 36c5b44f00..3f48be0c4d 100644
--- a/datafusion/proto/src/logical_plan/from_proto.rs
+++ b/datafusion/proto/src/logical_plan/from_proto.rs
@@ -27,6 +27,7 @@ use crate::protobuf::{
     OptimizedPhysicalPlanType, PlaceholderNode, RollupNode,
 };
 use arrow::{
+    array::AsArray,
     buffer::Buffer,
     datatypes::{
         i256, DataType, Field, IntervalMonthDayNanoType, IntervalUnit, Schema, TimeUnit,
@@ -722,9 +723,15 @@ impl TryFrom<&protobuf::ScalarValue> for ScalarValue {
                 .map_err(|e| e.context("Decoding ScalarValue::List Value"))?;
                 let arr = record_batch.column(0);
                 match value {
-                    Value::ListValue(_) => Self::List(arr.to_owned()),
-                    Value::LargeListValue(_) => Self::LargeList(arr.to_owned()),
-                    Value::FixedSizeListValue(_) => Self::FixedSizeList(arr.to_owned()),
+                    Value::ListValue(_) => {
+                        Self::List(arr.as_list::<i32>().to_owned().into())
+                    }
+                    Value::LargeListValue(_) => {
+                        Self::LargeList(arr.as_list::<i64>().to_owned().into())
+                    }
+                    Value::FixedSizeListValue(_) => {
+                        Self::FixedSizeList(arr.as_fixed_size_list().to_owned().into())
+                    }
                     _ => unreachable!(),
                 }
             }
diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rs
index a162b2389c..b7d1ef2252 100644
--- a/datafusion/proto/src/logical_plan/to_proto.rs
+++ b/datafusion/proto/src/logical_plan/to_proto.rs
@@ -32,6 +32,7 @@ use crate::protobuf::{
     OptimizedLogicalPlanType, OptimizedPhysicalPlanType, PlaceholderNode, RollupNode,
 };
 use arrow::{
+    array::ArrayRef,
     datatypes::{
         DataType, Field, IntervalMonthDayNanoType, IntervalUnit, Schema, SchemaRef,
         TimeUnit, UnionMode,
@@ -1159,54 +1160,15 @@ impl TryFrom<&ScalarValue> for protobuf::ScalarValue {
             }
             // ScalarValue::List and ScalarValue::FixedSizeList are serialized using
             // Arrow IPC messages as a single column RecordBatch
-            ScalarValue::List(arr)
-            | ScalarValue::LargeList(arr)
-            | ScalarValue::FixedSizeList(arr) => {
+            ScalarValue::List(arr) => {
+                encode_scalar_list_value(arr.to_owned() as ArrayRef, val)
+            }
+            ScalarValue::LargeList(arr) => {
                 // Wrap in a "field_name" column
-                let batch = RecordBatch::try_from_iter(vec![(
-                    "field_name",
-                    arr.to_owned(),
-                )])
-                .map_err(|e| {
-                   Error::General( format!("Error creating temporary batch while encoding ScalarValue::List: {e}"))
-                })?;
-
-                let gen = IpcDataGenerator {};
-                let mut dict_tracker = DictionaryTracker::new(false);
-                let (_, encoded_message) = gen
-                    .encoded_batch(&batch, &mut dict_tracker, &Default::default())
-                    .map_err(|e| {
-                        Error::General(format!(
-                            "Error encoding ScalarValue::List as IPC: {e}"
-                        ))
-                    })?;
-
-                let schema: protobuf::Schema = batch.schema().try_into()?;
-
-                let scalar_list_value = protobuf::ScalarListValue {
-                    ipc_message: encoded_message.ipc_message,
-                    arrow_data: encoded_message.arrow_data,
-                    schema: Some(schema),
-                };
-
-                match val {
-                    ScalarValue::List(_) => Ok(protobuf::ScalarValue {
-                        value: Some(protobuf::scalar_value::Value::ListValue(
-                            scalar_list_value,
-                        )),
-                    }),
-                    ScalarValue::LargeList(_) => Ok(protobuf::ScalarValue {
-                        value: Some(protobuf::scalar_value::Value::LargeListValue(
-                            scalar_list_value,
-                        )),
-                    }),
-                    ScalarValue::FixedSizeList(_) => Ok(protobuf::ScalarValue {
-                        value: Some(protobuf::scalar_value::Value::FixedSizeListValue(
-                            scalar_list_value,
-                        )),
-                    }),
-                    _ => unreachable!(),
-                }
+                encode_scalar_list_value(arr.to_owned() as ArrayRef, val)
+            }
+            ScalarValue::FixedSizeList(arr) => {
+                encode_scalar_list_value(arr.to_owned() as ArrayRef, val)
             }
             ScalarValue::Date32(val) => {
                 create_proto_scalar(val.as_ref(), &data_type, |s| Value::Date32Value(*s))
@@ -1723,3 +1685,47 @@ fn create_proto_scalar<I, T: FnOnce(&I) -> protobuf::scalar_value::Value>(
 
     Ok(protobuf::ScalarValue { value: Some(value) })
 }
+
+fn encode_scalar_list_value(
+    arr: ArrayRef,
+    val: &ScalarValue,
+) -> Result<protobuf::ScalarValue, Error> {
+    let batch = RecordBatch::try_from_iter(vec![("field_name", arr)]).map_err(|e| {
+        Error::General(format!(
+            "Error creating temporary batch while encoding ScalarValue::List: {e}"
+        ))
+    })?;
+
+    let gen = IpcDataGenerator {};
+    let mut dict_tracker = DictionaryTracker::new(false);
+    let (_, encoded_message) = gen
+        .encoded_batch(&batch, &mut dict_tracker, &Default::default())
+        .map_err(|e| {
+            Error::General(format!("Error encoding ScalarValue::List as IPC: {e}"))
+        })?;
+
+    let schema: protobuf::Schema = batch.schema().try_into()?;
+
+    let scalar_list_value = protobuf::ScalarListValue {
+        ipc_message: encoded_message.ipc_message,
+        arrow_data: encoded_message.arrow_data,
+        schema: Some(schema),
+    };
+
+    match val {
+        ScalarValue::List(_) => Ok(protobuf::ScalarValue {
+            value: Some(protobuf::scalar_value::Value::ListValue(scalar_list_value)),
+        }),
+        ScalarValue::LargeList(_) => Ok(protobuf::ScalarValue {
+            value: Some(protobuf::scalar_value::Value::LargeListValue(
+                scalar_list_value,
+            )),
+        }),
+        ScalarValue::FixedSizeList(_) => Ok(protobuf::ScalarValue {
+            value: Some(protobuf::scalar_value::Value::FixedSizeListValue(
+                scalar_list_value,
+            )),
+        }),
+        _ => unreachable!(),
+    }
+}