(asterixdb) branch master updated: [ASTERIXDB-3392][EXT] Fix false warnings while querying parquet

[mb...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 69cf54ce6d [ASTERIXDB-3392][EXT] Fix false warnings while querying parquet
69cf54ce6d is described below

commit 69cf54ce6d5224578003348a6670877e8cb0fa69
Author: preetham0202 <pr...@couchbase.com>
AuthorDate: Mon Jan 12 18:23:04 2026 +0530

    [ASTERIXDB-3392][EXT] Fix false warnings while querying parquet
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    Details:
    When querying Parquet files, we were seeing false warning counts, which this patch fixes. It also corrects the Parquet file naming format. Previously, files were named .parquet.zstd; they are now correctly named .zstd.parquet
    
    Ext-ref: MB-70108
    
    Change-Id: Id2dc25a30ea1bf7012f945803befc2751f33b86a
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20788
    Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Hussain Towaileb <hu...@gmail.com>
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Preetham Poluparthi <pr...@apache.org>
---
 .../org/apache/asterix/external/input/HDFSDataSourceFactory.java     | 5 ++---
 .../org/apache/asterix/metadata/provider/ExternalWriterProvider.java | 5 +++++
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
index b820147c93..82653c22a5 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
@@ -321,9 +321,8 @@ public class HDFSDataSourceFactory implements IRecordReaderFactory<Object>, IExt
             }
             restoreConfig(ctx);
             JobConf readerConf = conf;
-            if (ctx.getWarningCollector().shouldWarn()
-                    && configuration.get(ExternalDataConstants.KEY_INPUT_FORMAT.trim())
-                            .equals(ExternalDataConstants.INPUT_FORMAT_PARQUET)) {
+            if (configuration.get(ExternalDataConstants.KEY_INPUT_FORMAT.trim())
+                    .equals(ExternalDataConstants.INPUT_FORMAT_PARQUET)) {
                 /*
                  * JobConf is used to pass warnings from the ParquetReadSupport to ParquetReader. As multiple
                  * partitions can issue different warnings, we might have a race condition on JobConf. Thus, we
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
index bdfffa0cdd..763a7a111f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
@@ -99,6 +99,11 @@ public class ExternalWriterProvider {
         Map<String, String> configuration = sink.getConfiguration();
         String format = getFormat(configuration);
         String compression = getCompression(configuration);
+        if (format.equalsIgnoreCase(ExternalDataConstants.FORMAT_PARQUET)) {
+            // Parquet file extension format is like .snappy.parquet
+            return (compression.isEmpty() ? "" : compression.toLowerCase() + ".")
+                    + ExternalDataConstants.FORMAT_PARQUET;
+        }
         return format + (compression.isEmpty() ? "" : "." + compression);
     }