You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by hu...@apache.org on 2026/10/21 21:50:02 UTC
(iceberg) branch main updated: Parquet: Fix UnsupportedOperationException for boolean filters (#18580)
This is an automated email from the ASF dual-hosted git repository.
huaxingao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 16bb7a9903 Parquet: Fix UnsupportedOperationException for boolean filters (#18580)
16bb7a9903 is described below
commit 16bb7a99033124b3eb669ee5b0ea28dfc980ead9
Author: Yusuf YILDIZ <yu...@gmail.com>
AuthorDate: Wed Oct 21 17:49:55 2026 -0400
Parquet: Fix UnsupportedOperationException for boolean filters (#18580)
* Parquet: Fix UnsupportedOperationException for boolean filters
ParquetFilters rejected Boolean literals in EQ/NOT_EQ predicates and
had no IS_NULL/NOT_NULL handling for boolean columns, so reads with
such filters failed before returning any rows.
Closes #17092
Co-authored-by: Eunbin Son <th...@naver.com>
* Parquet: Define the schema locally in boolean filter tests
Follows the pattern of the other tests in TestParquet.
---------
Co-authored-by: Eunbin Son <th...@naver.com>
---
.../org/apache/iceberg/parquet/ParquetFilters.java | 6 ++-
.../org/apache/iceberg/parquet/TestParquet.java | 60 ++++++++++++++++++++--
2 files changed, 62 insertions(+), 4 deletions(-)
diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetFilters.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetFilters.java
index 39ae038afa..4f5d22e6ab 100644
--- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetFilters.java
+++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetFilters.java
@@ -131,6 +131,10 @@ class ParquetFilters {
case BOOLEAN:
Operators.BooleanColumn col = FilterApi.booleanColumn(path);
switch (op) {
+ case IS_NULL:
+ return FilterApi.eq(col, null);
+ case NOT_NULL:
+ return FilterApi.notEq(col, null);
case EQ:
return FilterApi.eq(col, getParquetPrimitive(lit));
case NOT_EQ:
@@ -223,7 +227,7 @@ class ParquetFilters {
// TODO: this needs to convert to handle BigDecimal and UUID
Object value = lit.value();
- if (value instanceof Number) {
+ if (value instanceof Number || value instanceof Boolean) {
return (C) lit.value();
} else if (value instanceof CharSequence) {
return (C) Binary.fromString(value.toString());
diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java
index b778536ae5..9440b46207 100644
--- a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java
+++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java
@@ -29,8 +29,12 @@ import static org.apache.iceberg.TableProperties.PARQUET_DICT_ENCODING_ENABLED_C
import static org.apache.iceberg.TableProperties.PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT;
import static org.apache.iceberg.TableProperties.PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT;
import static org.apache.iceberg.TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES;
+import static org.apache.iceberg.expressions.Expressions.equal;
import static org.apache.iceberg.expressions.Expressions.greaterThan;
import static org.apache.iceberg.expressions.Expressions.greaterThanOrEqual;
+import static org.apache.iceberg.expressions.Expressions.isNull;
+import static org.apache.iceberg.expressions.Expressions.notEqual;
+import static org.apache.iceberg.expressions.Expressions.notNull;
import static org.apache.iceberg.parquet.ParquetWritingTestUtils.createTempFile;
import static org.apache.iceberg.parquet.ParquetWritingTestUtils.write;
import static org.apache.iceberg.relocated.com.google.common.collect.Iterables.getOnlyElement;
@@ -803,7 +807,7 @@ public class TestParquet {
ImmutableMap.of(
"id", 5L, "ts", LocalDateTime.parse("2024-01-01T00:00:00.000003000"))));
- File file = writeNanoRecords(schema, records);
+ File file = writeRecords(schema, records);
// Boundary at 500 ns: only ids 3 (750 ns), 4 (1500 ns), 5 (3000 ns) qualify.
List<Long> ids =
@@ -841,7 +845,7 @@ public class TestParquet {
ImmutableMap.of(
"id", 5L, "ts", OffsetDateTime.parse("2024-01-01T00:00:00.000003000+00:00"))));
- File file = writeNanoRecords(schema, records);
+ File file = writeRecords(schema, records);
// Boundary == 2024-01-01T00:00:00.000000500Z, expressed in +04:00. id2 is that same instant
// (written in +05:00); a strict greaterThan excludes it, leaving ids 3/4/5.
@@ -850,7 +854,57 @@ public class TestParquet {
assertThat(ids).containsExactlyInAnyOrder(3L, 4L, 5L);
}
- private File writeNanoRecords(Schema schema, List<Record> records) throws IOException {
+ @Test
+ public void booleanEqualFilter() throws IOException {
+ Schema schema =
+ new Schema(
+ required(1, "id", Types.LongType.get()), optional(2, "flag", Types.BooleanType.get()));
+ File file = writeBooleanRecords(schema, true, false, true);
+
+ assertThat(filterIds(schema, file, equal("flag", true))).containsExactlyInAnyOrder(1L, 3L);
+ }
+
+ @Test
+ public void booleanNotEqualFilter() throws IOException {
+ Schema schema =
+ new Schema(
+ required(1, "id", Types.LongType.get()), optional(2, "flag", Types.BooleanType.get()));
+ File file = writeBooleanRecords(schema, true, false, true);
+
+ assertThat(filterIds(schema, file, notEqual("flag", true))).containsExactly(2L);
+ }
+
+ @Test
+ public void booleanIsNullFilter() throws IOException {
+ Schema schema =
+ new Schema(
+ required(1, "id", Types.LongType.get()), optional(2, "flag", Types.BooleanType.get()));
+ File file = writeBooleanRecords(schema, true, null, false);
+
+ assertThat(filterIds(schema, file, isNull("flag"))).containsExactly(2L);
+ }
+
+ @Test
+ public void booleanNotNullFilter() throws IOException {
+ Schema schema =
+ new Schema(
+ required(1, "id", Types.LongType.get()), optional(2, "flag", Types.BooleanType.get()));
+ File file = writeBooleanRecords(schema, true, null, false);
+
+ assertThat(filterIds(schema, file, notNull("flag"))).containsExactlyInAnyOrder(1L, 3L);
+ }
+
+ private File writeBooleanRecords(Schema schema, Boolean... flags) throws IOException {
+ Record template = org.apache.iceberg.data.GenericRecord.create(schema);
+ List<Record> records = Lists.newArrayList();
+ for (int i = 0; i < flags.length; i += 1) {
+ records.add(template.copy("id", (long) (i + 1), "flag", flags[i]));
+ }
+
+ return writeRecords(schema, records);
+ }
+
+ private File writeRecords(Schema schema, List<Record> records) throws IOException {
File file = createTempFile(temp);
try (FileAppender<Record> appender =
Parquet.write(Files.localOutput(file))