[GitHub] [arrow-datafusion] alamb commented on a diff in pull request #2960: test: add test for decimal and pruning for decimal column

[GitBox <gi...@apache.org>]

alamb commented on code in PR #2960:
URL: https://github.com/apache/arrow-datafusion/pull/2960#discussion_r929238797


##########
datafusion/core/src/datasource/file_format/parquet.rs:
##########
@@ -1023,6 +1023,49 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn read_decimal_parquet() -> Result<()> {
+        let session_ctx = SessionContext::new();
+        let task_ctx = session_ctx.task_ctx();
+
+        // parquet use the int32 as the physical type to store decimal
+        let exec = get_exec("int32_decimal.parquet", None, None).await?;
+        let batches = collect(exec, task_ctx.clone()).await?;
+        assert_eq!(1, batches.len());
+        assert_eq!(1, batches[0].num_columns());
+        let column = batches[0].column(0);
+        assert_eq!(&DataType::Decimal(4, 2), column.data_type());
+
+        // parquet use the int64 as the physical type to store decimal
+        let exec = get_exec("int64_decimal.parquet", None, None).await?;
+        let batches = collect(exec, task_ctx.clone()).await?;
+        assert_eq!(1, batches.len());
+        assert_eq!(1, batches[0].num_columns());
+        let column = batches[0].column(0);
+        assert_eq!(&DataType::Decimal(10, 2), column.data_type());
+
+        // parquet use the fixed length binary as the physical type to store decimal
+        let exec = get_exec("fixed_length_decimal.parquet", None, None).await?;
+        let batches = collect(exec, task_ctx.clone()).await?;
+        assert_eq!(1, batches.len());
+        assert_eq!(1, batches[0].num_columns());
+        let column = batches[0].column(0);
+        assert_eq!(&DataType::Decimal(25, 2), column.data_type());
+
+        let exec = get_exec("fixed_length_decimal_legacy.parquet", None, None).await?;
+        let batches = collect(exec, task_ctx.clone()).await?;
+        assert_eq!(1, batches.len());
+        assert_eq!(1, batches[0].num_columns());
+        let column = batches[0].column(0);
+        assert_eq!(&DataType::Decimal(13, 2), column.data_type());
+
+        // parquet use the fixed length binary as the physical type to store decimal
+        // TODO: arrow-rs don't support convert the physical type of binary to decimal

Review Comment:
   ```suggestion
           // TODO: arrow-rs don't support convert the physical type of binary to decimal
           // https://github.com/apache/arrow-rs/pull/2160
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org