You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by pa...@apache.org on 2025/04/17 20:49:07 UTC

(datafusion-comet) branch main updated: fix: handle missing field correctly in native_iceberg_compat (#1656)

This is an automated email from the ASF dual-hosted git repository.

parthc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git


The following commit(s) were added to refs/heads/main by this push:
     new e7a321451 fix: handle missing field correctly in native_iceberg_compat (#1656)
e7a321451 is described below

commit e7a3214510091cf9d177fabdf9e3221317d1e785
Author: Parth Chandra <pa...@apache.org>
AuthorDate: Thu Apr 17 13:49:04 2025 -0700

    fix: handle missing field correctly in native_iceberg_compat (#1656)
---
 .../main/java/org/apache/comet/parquet/BatchReader.java |  3 +++
 .../org/apache/comet/parquet/NativeBatchReader.java     | 17 +++++++++++++----
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/common/src/main/java/org/apache/comet/parquet/BatchReader.java b/common/src/main/java/org/apache/comet/parquet/BatchReader.java
index 914aafdaa..8bcbcc0d0 100644
--- a/common/src/main/java/org/apache/comet/parquet/BatchReader.java
+++ b/common/src/main/java/org/apache/comet/parquet/BatchReader.java
@@ -282,6 +282,9 @@ public class BatchReader extends RecordReader<Void, ColumnarBatch> implements Cl
     // Initialize missing columns and use null vectors for them
     missingColumns = new boolean[columns.size()];
     List<String[]> paths = requestedSchema.getPaths();
+    // We do not need the column index of the row index; but this method has the
+    // side effect of throwing an exception if a column with the same name is
+    // found which we do want (spark unit tests explicitly test for that).
     ShimFileFormat.findRowIndexColumnIndexInSchema(sparkSchema);
     StructField[] nonPartitionFields = sparkSchema.fields();
     for (int i = 0; i < requestedSchema.getFieldCount(); i++) {
diff --git a/common/src/main/java/org/apache/comet/parquet/NativeBatchReader.java b/common/src/main/java/org/apache/comet/parquet/NativeBatchReader.java
index dea515055..121cbe8d6 100644
--- a/common/src/main/java/org/apache/comet/parquet/NativeBatchReader.java
+++ b/common/src/main/java/org/apache/comet/parquet/NativeBatchReader.java
@@ -289,6 +289,9 @@ public class NativeBatchReader extends RecordReader<Void, ColumnarBatch> impleme
 
       // Initialize missing columns and use null vectors for them
       missingColumns = new boolean[numColumns];
+      // We do not need the column index of the row index; but this method has the
+      // side effect of throwing an exception if a column with the same name is
+      // found which we do want (spark unit tests explicitly test for that).
       ShimFileFormat.findRowIndexColumnIndexInSchema(sparkSchema);
       StructField[] nonPartitionFields = sparkSchema.fields();
       boolean hasRowIndexColumn = false;
@@ -342,10 +345,16 @@ public class NativeBatchReader extends RecordReader<Void, ColumnarBatch> impleme
                     + " in data file "
                     + filePath);
           }
-          ConstantColumnReader reader =
-              new ConstantColumnReader(nonPartitionFields[i], capacity, useDecimal128);
-          columnReaders[i] = reader;
-          missingColumns[i] = true;
+          if (field.isPrimitive()) {
+            ConstantColumnReader reader =
+                new ConstantColumnReader(nonPartitionFields[i], capacity, useDecimal128);
+            columnReaders[i] = reader;
+            missingColumns[i] = true;
+          } else {
+            // the column requested is not in the file, but the native reader can handle that
+            // and will return nulls for all rows requested
+            missingColumns[i] = false;
+          }
         }
       }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org