You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2019/02/24 19:03:40 UTC
[incubator-iceberg] branch master updated: Fix Iceberg Parquet
Reader scanning when filtering on nested types (#110)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 3c72c06 Fix Iceberg Parquet Reader scanning when filtering on nested types (#110)
3c72c06 is described below
commit 3c72c06fa4b48aa827faccbae7e7a7b799059111
Author: Gautam <ga...@gmail.com>
AuthorDate: Mon Feb 25 00:33:35 2019 +0530
Fix Iceberg Parquet Reader scanning when filtering on nested types (#110)
* Fix parquet reader scan not returning rows when filtering on nested columns
* Addressed review comments, added unit tests for map & struct types for isNull & notNull expressions
---
.../parquet/ParquetMetricsRowGroupFilter.java | 15 +++++++++++++
.../iceberg/parquet/TestMetricsRowGroupFilter.java | 25 +++++++++++++++++++++-
2 files changed, 39 insertions(+), 1 deletion(-)
diff --git a/parquet/src/main/java/com/netflix/iceberg/parquet/ParquetMetricsRowGroupFilter.java b/parquet/src/main/java/com/netflix/iceberg/parquet/ParquetMetricsRowGroupFilter.java
index 490eb16..1961d9b 100644
--- a/parquet/src/main/java/com/netflix/iceberg/parquet/ParquetMetricsRowGroupFilter.java
+++ b/parquet/src/main/java/com/netflix/iceberg/parquet/ParquetMetricsRowGroupFilter.java
@@ -28,6 +28,7 @@ import com.netflix.iceberg.expressions.Expression;
import com.netflix.iceberg.expressions.ExpressionVisitors;
import com.netflix.iceberg.expressions.ExpressionVisitors.BoundExpressionVisitor;
import com.netflix.iceberg.expressions.Literal;
+import com.netflix.iceberg.types.Type;
import com.netflix.iceberg.types.Types;
import com.netflix.iceberg.types.Types.StructType;
import org.apache.parquet.column.statistics.Statistics;
@@ -156,6 +157,13 @@ public class ParquetMetricsRowGroupFilter {
Preconditions.checkNotNull(struct.field(id),
"Cannot filter by nested column: %s", schema.findField(id));
+ // When filtering nested types notNull() is implicit filter passed even though complex
+ // filters aren't pushed down in Parquet. Leave all nested column type filters to be
+ // evaluated post scan.
+ if (schema.findType(id) instanceof Type.NestedType) {
+ return ROWS_MIGHT_MATCH;
+ }
+
Long valueCount = valueCounts.get(id);
if (valueCount == null) {
// the column is not present and is all nulls
@@ -289,6 +297,13 @@ public class ParquetMetricsRowGroupFilter {
Types.NestedField field = struct.field(id);
Preconditions.checkNotNull(field, "Cannot filter by nested column: %s", schema.findField(id));
+ // When filtering nested types notNull() is implicit filter passed even though complex
+ // filters aren't pushed down in Parquet. Leave all nested column type filters to be
+ // evaluated post scan.
+ if (schema.findType(id) instanceof Type.NestedType) {
+ return ROWS_MIGHT_MATCH;
+ }
+
Long valueCount = valueCounts.get(id);
if (valueCount == null) {
// the column is not present and is all nulls
diff --git a/parquet/src/test/java/com/netflix/iceberg/parquet/TestMetricsRowGroupFilter.java b/parquet/src/test/java/com/netflix/iceberg/parquet/TestMetricsRowGroupFilter.java
index d31d366..fc2c418 100644
--- a/parquet/src/test/java/com/netflix/iceberg/parquet/TestMetricsRowGroupFilter.java
+++ b/parquet/src/test/java/com/netflix/iceberg/parquet/TestMetricsRowGroupFilter.java
@@ -27,6 +27,7 @@ import com.netflix.iceberg.expressions.Expression;
import com.netflix.iceberg.io.FileAppender;
import com.netflix.iceberg.io.InputFile;
import com.netflix.iceberg.io.OutputFile;
+import com.netflix.iceberg.types.Types;
import com.netflix.iceberg.types.Types.FloatType;
import com.netflix.iceberg.types.Types.IntegerType;
import com.netflix.iceberg.types.Types.LongType;
@@ -42,6 +43,8 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
import java.util.UUID;
import static com.netflix.iceberg.avro.AvroSchemaUtil.convert;
@@ -67,7 +70,11 @@ public class TestMetricsRowGroupFilter {
optional(4, "all_nulls", LongType.get()),
optional(5, "some_nulls", StringType.get()),
optional(6, "no_nulls", StringType.get()),
- optional(7, "not_in_file", FloatType.get())
+ optional(7, "not_in_file", FloatType.get()),
+ optional(8, "map_not_null",
+ Types.MapType.ofRequired(9, 10, StringType.get(), IntegerType.get())),
+ optional(9, "struct_not_null",
+ Types.StructType.of(Types.NestedField.required(10, "struct_subfield", StringType.get())))
);
private static final Schema FILE_SCHEMA = new Schema(
@@ -138,6 +145,14 @@ public class TestMetricsRowGroupFilter {
shouldRead = new ParquetMetricsRowGroupFilter(SCHEMA, notNull("no_nulls"))
.shouldRead(PARQUET_SCHEMA, ROW_GROUP_METADATA);
Assert.assertTrue("Should read: non-null column contains a non-null value", shouldRead);
+
+ shouldRead = new ParquetMetricsRowGroupFilter(SCHEMA, notNull("map_not_null"))
+ .shouldRead(PARQUET_SCHEMA, ROW_GROUP_METADATA);
+ Assert.assertTrue("Should read: map type is not skipped", shouldRead);
+
+ shouldRead = new ParquetMetricsRowGroupFilter(SCHEMA, notNull("struct_not_null"))
+ .shouldRead(PARQUET_SCHEMA, ROW_GROUP_METADATA);
+ Assert.assertTrue("Should read: struct type is not skipped", shouldRead);
}
@Test
@@ -153,6 +168,14 @@ public class TestMetricsRowGroupFilter {
shouldRead = new ParquetMetricsRowGroupFilter(SCHEMA, isNull("no_nulls"))
.shouldRead(PARQUET_SCHEMA, ROW_GROUP_METADATA);
Assert.assertFalse("Should skip: non-null column contains no null values", shouldRead);
+
+ shouldRead = new ParquetMetricsRowGroupFilter(SCHEMA, isNull("map_not_null"))
+ .shouldRead(PARQUET_SCHEMA, ROW_GROUP_METADATA);
+ Assert.assertTrue("Should read: map type is not skipped", shouldRead);
+
+ shouldRead = new ParquetMetricsRowGroupFilter(SCHEMA, isNull("struct_not_null"))
+ .shouldRead(PARQUET_SCHEMA, ROW_GROUP_METADATA);
+ Assert.assertTrue("Should read: struct type is not skipped", shouldRead);
}
@Test