You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by pa...@apache.org on 2025/04/17 20:49:07 UTC
(datafusion-comet) branch main updated: fix: handle missing field correctly in native_iceberg_compat (#1656)
This is an automated email from the ASF dual-hosted git repository.
parthc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new e7a321451 fix: handle missing field correctly in native_iceberg_compat (#1656)
e7a321451 is described below
commit e7a3214510091cf9d177fabdf9e3221317d1e785
Author: Parth Chandra <pa...@apache.org>
AuthorDate: Thu Apr 17 13:49:04 2025 -0700
fix: handle missing field correctly in native_iceberg_compat (#1656)
---
.../main/java/org/apache/comet/parquet/BatchReader.java | 3 +++
.../org/apache/comet/parquet/NativeBatchReader.java | 17 +++++++++++++----
2 files changed, 16 insertions(+), 4 deletions(-)
diff --git a/common/src/main/java/org/apache/comet/parquet/BatchReader.java b/common/src/main/java/org/apache/comet/parquet/BatchReader.java
index 914aafdaa..8bcbcc0d0 100644
--- a/common/src/main/java/org/apache/comet/parquet/BatchReader.java
+++ b/common/src/main/java/org/apache/comet/parquet/BatchReader.java
@@ -282,6 +282,9 @@ public class BatchReader extends RecordReader<Void, ColumnarBatch> implements Cl
// Initialize missing columns and use null vectors for them
missingColumns = new boolean[columns.size()];
List<String[]> paths = requestedSchema.getPaths();
+ // We do not need the column index of the row index; but this method has the
+ // side effect of throwing an exception if a column with the same name is
+ // found which we do want (spark unit tests explicitly test for that).
ShimFileFormat.findRowIndexColumnIndexInSchema(sparkSchema);
StructField[] nonPartitionFields = sparkSchema.fields();
for (int i = 0; i < requestedSchema.getFieldCount(); i++) {
diff --git a/common/src/main/java/org/apache/comet/parquet/NativeBatchReader.java b/common/src/main/java/org/apache/comet/parquet/NativeBatchReader.java
index dea515055..121cbe8d6 100644
--- a/common/src/main/java/org/apache/comet/parquet/NativeBatchReader.java
+++ b/common/src/main/java/org/apache/comet/parquet/NativeBatchReader.java
@@ -289,6 +289,9 @@ public class NativeBatchReader extends RecordReader<Void, ColumnarBatch> impleme
// Initialize missing columns and use null vectors for them
missingColumns = new boolean[numColumns];
+ // We do not need the column index of the row index; but this method has the
+ // side effect of throwing an exception if a column with the same name is
+ // found which we do want (spark unit tests explicitly test for that).
ShimFileFormat.findRowIndexColumnIndexInSchema(sparkSchema);
StructField[] nonPartitionFields = sparkSchema.fields();
boolean hasRowIndexColumn = false;
@@ -342,10 +345,16 @@ public class NativeBatchReader extends RecordReader<Void, ColumnarBatch> impleme
+ " in data file "
+ filePath);
}
- ConstantColumnReader reader =
- new ConstantColumnReader(nonPartitionFields[i], capacity, useDecimal128);
- columnReaders[i] = reader;
- missingColumns[i] = true;
+ if (field.isPrimitive()) {
+ ConstantColumnReader reader =
+ new ConstantColumnReader(nonPartitionFields[i], capacity, useDecimal128);
+ columnReaders[i] = reader;
+ missingColumns[i] = true;
+ } else {
+ // the column requested is not in the file, but the native reader can handle that
+ // and will return nulls for all rows requested
+ missingColumns[i] = false;
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org