(datafusion) branch main updated: writing table to parquet followed by read and schema check (#21444)

[gi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

github-bot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git


The following commit(s) were added to refs/heads/main by this push:
     new 5ba06ac94e writing table to parquet followed by read and schema check (#21444)
5ba06ac94e is described below

commit 5ba06ac94ef5aea6796200e429fe551216c9c68d
Author: RIchard Baah <13...@users.noreply.github.com>
AuthorDate: Wed Apr 8 07:32:38 2026 -0400

    writing table to parquet followed by read and schema check (#21444)
    
    ## Which issue does this PR close?
    This attempts to bridge the missing test coverage mentioned by @alamb on
    this issue https://github.com/apache/datafusion/issues/8791
    <!--
    We generally require a GitHub issue to be filed for all bug fixes and
    enhancements and this helps us generate change logs for our releases.
    You can link an issue to this PR using the GitHub syntax. For example
    `Closes #123` indicates that this PR will close issue #123.
    -->
    
    - Closes #.
    
    ## Rationale for this change
    
    <!--
    Why are you proposing this change? If this is already explained clearly
    in the issue then this section is not needed.
    Explaining clearly why changes are proposed helps reviewers understand
    your changes and offer better suggestions for fixes.
    -->
    
    ## What changes are included in this PR?
    
    <!--
    There is no need to duplicate the description in the issue here but it
    is sometimes worth providing a summary of the individual changes in this
    PR.
    -->
    
    ## Are these changes tested?
    The changes are test
    <!--
    We typically require tests for all PRs in order to:
    1. Prevent the code from being accidentally broken by subsequent changes
    2. Serve as another way to document the expected behavior of the code
    
    If tests are not included in your PR, please explain why (for example,
    are they covered by existing tests)?
    -->
    
    ## Are there any user-facing changes?
    no
    <!--
    If there are user-facing changes then we may require documentation to be
    updated before approving the PR.
    -->
    
    <!--
    If there are any breaking changes to public APIs, please add the `api
    change` label.
    -->
---
 datafusion/sqllogictest/test_files/dictionary.slt | 60 +++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/datafusion/sqllogictest/test_files/dictionary.slt b/datafusion/sqllogictest/test_files/dictionary.slt
index 5ec95260e8..92e6c41835 100644
--- a/datafusion/sqllogictest/test_files/dictionary.slt
+++ b/datafusion/sqllogictest/test_files/dictionary.slt
@@ -455,3 +455,63 @@ CREATE TABLE test0 AS VALUES ('foo',1), ('bar',2), ('foo',3);
 
 statement ok
 COPY (SELECT arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS column1, column2 FROM test0) TO 'test_files/scratch/dictionary/part_dict_test' STORED AS PARQUET PARTITIONED BY (column1);
+
+
+# Dictionary-encoded large hashes repeated across 10 rows (3 unique hashes)
+statement ok
+CREATE TABLE dict_hash_src AS
+SELECT
+  arrow_cast(column1, 'Int64') AS id,
+  arrow_cast(column2, 'Dictionary(Int32, Utf8)') AS payload_hash,
+  arrow_cast(column3, 'Float64') AS metric,
+  arrow_cast(column4, 'Timestamp(ns)') AS ts
+FROM (
+  VALUES
+    (1,  '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08', 10.0, 1703030400000000000),
+    (2,  '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08', 11.0, 1703031000000000000),
+    (3,  '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08', 12.0, 1703031600000000000),
+    (4,  '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08', 13.0, 1703032200000000000),
+    (5,  '1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f53', 14.0, 1703032800000000000),
+    (6,  '1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f53', 15.0, 1703033400000000000),
+    (7,  '1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f53', 16.0, 1703034000000000000),
+    (8,  '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b', 17.0, 1703034600000000000),
+    (9,  '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b', 18.0, 1703035200000000000),
+    (10, '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b', 19.0, 1703035800000000000)
+);
+
+statement ok
+COPY dict_hash_src
+TO 'test_files/scratch/dictionary/dict_hash_10.parquet'
+STORED AS PARQUET;
+
+statement ok
+CREATE EXTERNAL TABLE dict_hash_10
+STORED AS PARQUET
+LOCATION 'test_files/scratch/dictionary/dict_hash_10.parquet';
+
+query TTT
+DESCRIBE dict_hash_10;
+----
+id Int64 YES
+payload_hash Dictionary(Int32, Utf8) YES
+metric Float64 YES
+ts Timestamp(ns) YES
+
+query II
+SELECT COUNT(*), COUNT(DISTINCT payload_hash)
+FROM dict_hash_10;
+----
+10 3
+
+query I
+SELECT COUNT(*)
+FROM dict_hash_10
+WHERE payload_hash = '1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f53';
+----
+3
+
+statement ok
+DROP TABLE dict_hash_10;
+
+statement ok
+DROP TABLE dict_hash_src;


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org