You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by sz...@apache.org on 2026/09/11 23:09:15 UTC

(iceberg) branch main updated: Parquet: Compute geometry bounding box metrics (#17161)

This is an automated email from the ASF dual-hosted git repository.

szehon-ho pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new b00a282b03 Parquet: Compute geometry bounding box metrics (#17161)
b00a282b03 is described below

commit b00a282b037ca4d43c1018a56ebafd69d4fa0e00
Author: Xin Huang <42...@users.noreply.github.com>
AuthorDate: Fri Sep 11 16:09:08 2026 -0700

    Parquet: Compute geometry bounding box metrics (#17161)
---
 .../org/apache/iceberg/GeometryFieldMetrics.java   |  83 +++++++++++++++++
 .../apache/iceberg/TestGeometryFieldMetrics.java   | 101 +++++++++++++++++++++
 .../test/java/org/apache/iceberg/TestMetrics.java  |  18 +++-
 .../iceberg/data/parquet/BaseParquetWriter.java    |   7 +-
 .../iceberg/parquet/ParquetValueWriters.java       |  30 ++++++
 .../iceberg/parquet/TestParquetValueWriters.java   |  27 ++++++
 6 files changed, 259 insertions(+), 7 deletions(-)

diff --git a/core/src/main/java/org/apache/iceberg/GeometryFieldMetrics.java b/core/src/main/java/org/apache/iceberg/GeometryFieldMetrics.java
new file mode 100644
index 0000000000..6a62b140bd
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/GeometryFieldMetrics.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.nio.ByteBuffer;
+import org.apache.iceberg.geospatial.BoundingBox;
+import org.apache.iceberg.geospatial.GeospatialBound;
+import org.apache.iceberg.types.Type;
+
+/**
+ * Field-level metrics for a geometry column, tracked by the Parquet writer.
+ *
+ * <p>The Parquet footer's lexicographic min/max over the WKB bytes is not meaningful for spatial
+ * values, so the writer instead scans each value's coordinates with a {@link GeometryBoundsBuilder}
+ * and accumulates a two-dimensional bounding box. The resulting {@link FieldMetrics} carries the
+ * box's lower and upper corners as {@link GeospatialBound} points, which serialize through the
+ * existing geometry conversion path. The null value count is left at zero here; it is reconciled by
+ * the optional-field writer that owns null accounting.
+ *
+ * <p>The average WKB size over the non-null values is also tracked, matching the {@code
+ * avg_value_size_in_bytes} metric that the counts-only geospatial writer produces, so adding bounds
+ * does not drop that metric for generic Parquet geometry writes.
+ */
+public class GeometryFieldMetrics extends FieldMetrics<GeospatialBound> {
+
+  private GeometryFieldMetrics(
+      int id,
+      long valueCount,
+      GeospatialBound lowerBound,
+      GeospatialBound upperBound,
+      Type type,
+      Integer avgValueSizeInBytes) {
+    super(id, valueCount, 0L, -1L, lowerBound, upperBound, type, avgValueSizeInBytes);
+  }
+
+  public static class Builder {
+    private final int id;
+    private final Type geometryType;
+    private final GeometryBoundsBuilder boundsBuilder = new GeometryBoundsBuilder();
+    private long valueCount = 0;
+    private long totalValueSizeInBytes = 0;
+
+    public Builder(int id, Type geometryType) {
+      this.id = id;
+      this.geometryType = geometryType;
+    }
+
+    public void addValue(ByteBuffer wkb) {
+      this.valueCount += 1;
+      this.totalValueSizeInBytes += wkb.remaining();
+      boundsBuilder.addValue(wkb);
+    }
+
+    public GeometryFieldMetrics build() {
+      // build() returns null bounds when a dimension has no value or a value could not be bounded,
+      // which leaves the field with counts but no bounds. The average size is still reported: it
+      // does not depend on whether a box could be formed.
+      BoundingBox box = boundsBuilder.build();
+      GeospatialBound lowerBound = box == null ? null : box.min();
+      GeospatialBound upperBound = box == null ? null : box.max();
+      Integer avgValueSizeInBytes =
+          valueCount > 0 ? Math.toIntExact(totalValueSizeInBytes / valueCount) : null;
+      return new GeometryFieldMetrics(
+          id, valueCount, lowerBound, upperBound, geometryType, avgValueSizeInBytes);
+    }
+  }
+}
diff --git a/core/src/test/java/org/apache/iceberg/TestGeometryFieldMetrics.java b/core/src/test/java/org/apache/iceberg/TestGeometryFieldMetrics.java
new file mode 100644
index 0000000000..953835b62e
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/TestGeometryFieldMetrics.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import org.apache.iceberg.geospatial.GeospatialBound;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.Test;
+
+public class TestGeometryFieldMetrics {
+
+  private static final Types.GeometryType GEOMETRY = Types.GeometryType.crs84();
+
+  @Test
+  public void testBoundsAcrossMultiplePoints() {
+    GeometryFieldMetrics.Builder builder = new GeometryFieldMetrics.Builder(2, GEOMETRY);
+    builder.addValue(wkbPoint(30, 10));
+    builder.addValue(wkbPoint(-5, 40));
+
+    FieldMetrics<GeospatialBound> metrics = builder.build();
+
+    assertThat(metrics.id()).isEqualTo(2);
+    assertThat(metrics.valueCount()).isEqualTo(2L);
+    // null accounting is reconciled by the optional-field writer, not here
+    assertThat(metrics.nullValueCount()).isEqualTo(0L);
+    // geometry has no NaN metric; -1 suppresses the entry downstream
+    assertThat(metrics.nanValueCount()).isEqualTo(-1L);
+    assertThat(metrics.originalType()).isEqualTo(GEOMETRY);
+    assertThat(metrics.lowerBound()).isEqualTo(GeospatialBound.createXY(-5, 10));
+    assertThat(metrics.upperBound()).isEqualTo(GeospatialBound.createXY(30, 40));
+    // both WKB points are 21 bytes, so the average WKB size is 21
+    assertThat(metrics.avgValueSizeInBytes()).isEqualTo(21);
+  }
+
+  @Test
+  public void testSinglePointBoundsAreThatPoint() {
+    GeometryFieldMetrics.Builder builder = new GeometryFieldMetrics.Builder(2, GEOMETRY);
+    builder.addValue(wkbPoint(12, 34));
+
+    FieldMetrics<GeospatialBound> metrics = builder.build();
+
+    assertThat(metrics.valueCount()).isEqualTo(1L);
+    assertThat(metrics.lowerBound()).isEqualTo(GeospatialBound.createXY(12, 34));
+    assertThat(metrics.upperBound()).isEqualTo(GeospatialBound.createXY(12, 34));
+    assertThat(metrics.avgValueSizeInBytes()).isEqualTo(21);
+  }
+
+  @Test
+  public void testEmptyGeometryProducesNoBounds() {
+    // POINT EMPTY (NaN coordinates) still counts as a value but contributes no coordinate.
+    GeometryFieldMetrics.Builder builder = new GeometryFieldMetrics.Builder(2, GEOMETRY);
+    builder.addValue(wkbPoint(Double.NaN, Double.NaN));
+
+    FieldMetrics<GeospatialBound> metrics = builder.build();
+
+    assertThat(metrics.valueCount()).isEqualTo(1L);
+    assertThat(metrics.lowerBound()).isNull();
+    assertThat(metrics.upperBound()).isNull();
+    // the value has no bounds but still has a size, so the average is still reported
+    assertThat(metrics.avgValueSizeInBytes()).isEqualTo(21);
+  }
+
+  @Test
+  public void testNoValuesProducesNoBounds() {
+    FieldMetrics<GeospatialBound> metrics = new GeometryFieldMetrics.Builder(2, GEOMETRY).build();
+
+    assertThat(metrics.valueCount()).isEqualTo(0L);
+    assertThat(metrics.lowerBound()).isNull();
+    assertThat(metrics.upperBound()).isNull();
+    assertThat(metrics.avgValueSizeInBytes()).isNull();
+  }
+
+  private static ByteBuffer wkbPoint(double xCoord, double yCoord) {
+    return ByteBuffer.allocate(21)
+        .order(ByteOrder.LITTLE_ENDIAN)
+        .put((byte) 1) // little-endian
+        .putInt(1) // WKB geometry type: Point
+        .putDouble(xCoord)
+        .putDouble(yCoord)
+        .flip();
+  }
+}
diff --git a/core/src/test/java/org/apache/iceberg/TestMetrics.java b/core/src/test/java/org/apache/iceberg/TestMetrics.java
index 3c68b0e39d..9b25825316 100644
--- a/core/src/test/java/org/apache/iceberg/TestMetrics.java
+++ b/core/src/test/java/org/apache/iceberg/TestMetrics.java
@@ -37,6 +37,7 @@ import java.util.List;
 import java.util.Map;
 import org.apache.iceberg.data.GenericRecord;
 import org.apache.iceberg.data.Record;
+import org.apache.iceberg.geospatial.GeospatialBound;
 import org.apache.iceberg.io.InputFile;
 import org.apache.iceberg.io.OutputFile;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
@@ -278,15 +279,22 @@ public abstract class TestMetrics {
     first.setField("geog", geog);
     Record second = GenericRecord.create(schema);
     second.setField("id", 2L);
-    // both geo columns are left null
+    second.setField("geom", wkbPoint(-5, 40));
+    // geog on the second row is left null
 
     Metrics metrics = getMetrics(schema, first, second);
     assertThat(metrics.recordCount()).isEqualTo(2L);
 
-    // geometry and geography keep value/null counts but no bounds: lexicographic WKB min/max is not
-    // meaningful, so bounds are intentionally skipped (spatial bounds are a separate follow-up).
-    assertCounts(2, 2L, 1L, metrics);
-    assertBounds(2, Types.GeometryType.crs84(), null, null, metrics);
+    // geometry bounds are the XY bounding box of all values: min corner (-5, 10), max corner
+    // (30, 40). The null geography row leaves geography with value/null counts but no bounds
+    // (geography bounds are a separate follow-up).
+    assertCounts(2, 2L, 0L, metrics);
+    assertBounds(
+        2,
+        Types.GeometryType.crs84(),
+        GeospatialBound.createXY(-5, 10),
+        GeospatialBound.createXY(30, 40),
+        metrics);
     assertCounts(3, 2L, 1L, metrics);
     assertBounds(3, Types.GeographyType.crs84(), null, null, metrics);
     assertThat(metrics.avgValueSizes())
diff --git a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java
index 78b924db5b..d14c05e0b2 100644
--- a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java
+++ b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java
@@ -268,8 +268,11 @@ abstract class BaseParquetWriter<T> {
     @Override
     public Optional<ParquetValueWriter<?>> visit(
         LogicalTypeAnnotation.GeometryLogicalTypeAnnotation geometryType) {
-      // geometry values are pure WKB stored in a BINARY column
-      return Optional.of(ParquetValueWriters.geospatial(desc));
+      // geometry values are pure WKB stored in a BINARY column; the writer also scans the
+      // coordinates to produce a bounding box (the Parquet footer cannot). The concrete CRS is
+      // immaterial here: ParquetMetrics rewrites the bound's type from the table schema before
+      // serialization, and the serialization is keyed on the type id, which ignores the CRS.
+      return Optional.of(ParquetValueWriters.geometry(desc, Types.GeometryType.crs84()));
     }
 
     @Override
diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueWriters.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueWriters.java
index 352e01d3f4..89992ecfa0 100644
--- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueWriters.java
+++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueWriters.java
@@ -36,6 +36,7 @@ import org.apache.avro.util.Utf8;
 import org.apache.iceberg.DoubleFieldMetrics;
 import org.apache.iceberg.FieldMetrics;
 import org.apache.iceberg.FloatFieldMetrics;
+import org.apache.iceberg.GeometryFieldMetrics;
 import org.apache.iceberg.StructLike;
 import org.apache.iceberg.ValueSizeFieldMetrics;
 import org.apache.iceberg.deletes.PositionDelete;
@@ -125,6 +126,11 @@ public class ParquetValueWriters {
     return new GeospatialWriter(desc);
   }
 
+  public static PrimitiveWriter<ByteBuffer> geometry(
+      ColumnDescriptor desc, org.apache.iceberg.types.Type geometryType) {
+    return new GeometryWriter(desc, geometryType);
+  }
+
   public static PrimitiveWriter<ByteBuffer> fixedBuffers(ColumnDescriptor desc) {
     return new FixedBufferWriter(desc);
   }
@@ -362,6 +368,30 @@ public class ParquetValueWriters {
     }
   }
 
+  private static class GeometryWriter extends PrimitiveWriter<ByteBuffer> {
+    private final GeometryFieldMetrics.Builder metricsBuilder;
+
+    private GeometryWriter(ColumnDescriptor desc, org.apache.iceberg.types.Type geometryType) {
+      super(desc);
+      this.metricsBuilder =
+          new GeometryFieldMetrics.Builder(
+              desc.getPrimitiveType().getId().intValue(), geometryType);
+    }
+
+    @Override
+    public void write(int repetitionLevel, ByteBuffer buffer) {
+      // Accumulate the bounding box before writing, so it reads the buffer's coordinates while the
+      // position is intact (the scanner reads a duplicate and leaves this buffer untouched).
+      metricsBuilder.addValue(buffer);
+      column.writeBinary(repetitionLevel, Binary.fromReusedByteBuffer(buffer));
+    }
+
+    @Override
+    public Stream<FieldMetrics<?>> metrics() {
+      return Stream.of(metricsBuilder.build());
+    }
+  }
+
   private static class FixedBufferWriter extends PrimitiveWriter<ByteBuffer> {
     private final int length;
 
diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetValueWriters.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetValueWriters.java
index 33c4044a0a..d5c968b117 100644
--- a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetValueWriters.java
+++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetValueWriters.java
@@ -60,4 +60,31 @@ class TestParquetValueWriters {
     assertThat(metrics.nullValueCount()).isEqualTo(1);
     assertThat(metrics.avgValueSizeInBytes()).isEqualTo(31);
   }
+
+  @Test
+  void geometryValueSizeMetricsExcludeNulls() {
+    Schema schema = new Schema(optional(2, "geom", Types.GeometryType.crs84()));
+    MessageType parquetSchema = ParquetSchemaUtil.convert(schema, "table");
+    Type parquetType = parquetSchema.getType("geom");
+    ColumnDescriptor desc = parquetSchema.getColumnDescription(new String[] {"geom"});
+    ParquetValueWriter<ByteBuffer> writer =
+        ParquetValueWriters.option(
+            parquetType,
+            parquetSchema.getMaxDefinitionLevel(new String[] {"geom"}),
+            ParquetValueWriters.geometry(desc, Types.GeometryType.crs84()));
+
+    ColumnWriteStore columnStore = mock(ColumnWriteStore.class);
+    when(columnStore.getColumnWriter(desc)).thenReturn(mock(ColumnWriter.class));
+    writer.setColumnStore(columnStore);
+    writer.write(0, ByteBuffer.allocate(21));
+    writer.write(0, ByteBuffer.allocate(42));
+    writer.write(0, null);
+
+    // the geometry writer adds bounds but must keep the same average WKB size metric as the
+    // counts-only geospatial writer: the average is over the two non-null values, (21 + 42) / 2
+    FieldMetrics<?> metrics = writer.metrics().findFirst().orElseThrow();
+    assertThat(metrics.valueCount()).isEqualTo(3);
+    assertThat(metrics.nullValueCount()).isEqualTo(1);
+    assertThat(metrics.avgValueSizeInBytes()).isEqualTo(31);
+  }
 }