You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by sz...@apache.org on 2026/09/11 23:09:15 UTC
(iceberg) branch main updated: Parquet: Compute geometry bounding box metrics (#17161)
This is an automated email from the ASF dual-hosted git repository.
szehon-ho pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new b00a282b03 Parquet: Compute geometry bounding box metrics (#17161)
b00a282b03 is described below
commit b00a282b037ca4d43c1018a56ebafd69d4fa0e00
Author: Xin Huang <42...@users.noreply.github.com>
AuthorDate: Fri Sep 11 16:09:08 2026 -0700
Parquet: Compute geometry bounding box metrics (#17161)
---
.../org/apache/iceberg/GeometryFieldMetrics.java | 83 +++++++++++++++++
.../apache/iceberg/TestGeometryFieldMetrics.java | 101 +++++++++++++++++++++
.../test/java/org/apache/iceberg/TestMetrics.java | 18 +++-
.../iceberg/data/parquet/BaseParquetWriter.java | 7 +-
.../iceberg/parquet/ParquetValueWriters.java | 30 ++++++
.../iceberg/parquet/TestParquetValueWriters.java | 27 ++++++
6 files changed, 259 insertions(+), 7 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/GeometryFieldMetrics.java b/core/src/main/java/org/apache/iceberg/GeometryFieldMetrics.java
new file mode 100644
index 0000000000..6a62b140bd
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/GeometryFieldMetrics.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.nio.ByteBuffer;
+import org.apache.iceberg.geospatial.BoundingBox;
+import org.apache.iceberg.geospatial.GeospatialBound;
+import org.apache.iceberg.types.Type;
+
+/**
+ * Field-level metrics for a geometry column, tracked by the Parquet writer.
+ *
+ * <p>The Parquet footer's lexicographic min/max over the WKB bytes is not meaningful for spatial
+ * values, so the writer instead scans each value's coordinates with a {@link GeometryBoundsBuilder}
+ * and accumulates a two-dimensional bounding box. The resulting {@link FieldMetrics} carries the
+ * box's lower and upper corners as {@link GeospatialBound} points, which serialize through the
+ * existing geometry conversion path. The null value count is left at zero here; it is reconciled by
+ * the optional-field writer that owns null accounting.
+ *
+ * <p>The average WKB size over the non-null values is also tracked, matching the {@code
+ * avg_value_size_in_bytes} metric that the counts-only geospatial writer produces, so adding bounds
+ * does not drop that metric for generic Parquet geometry writes.
+ */
+public class GeometryFieldMetrics extends FieldMetrics<GeospatialBound> {
+
+ private GeometryFieldMetrics(
+ int id,
+ long valueCount,
+ GeospatialBound lowerBound,
+ GeospatialBound upperBound,
+ Type type,
+ Integer avgValueSizeInBytes) {
+ super(id, valueCount, 0L, -1L, lowerBound, upperBound, type, avgValueSizeInBytes);
+ }
+
+ public static class Builder {
+ private final int id;
+ private final Type geometryType;
+ private final GeometryBoundsBuilder boundsBuilder = new GeometryBoundsBuilder();
+ private long valueCount = 0;
+ private long totalValueSizeInBytes = 0;
+
+ public Builder(int id, Type geometryType) {
+ this.id = id;
+ this.geometryType = geometryType;
+ }
+
+ public void addValue(ByteBuffer wkb) {
+ this.valueCount += 1;
+ this.totalValueSizeInBytes += wkb.remaining();
+ boundsBuilder.addValue(wkb);
+ }
+
+ public GeometryFieldMetrics build() {
+ // build() returns null bounds when a dimension has no value or a value could not be bounded,
+ // which leaves the field with counts but no bounds. The average size is still reported: it
+ // does not depend on whether a box could be formed.
+ BoundingBox box = boundsBuilder.build();
+ GeospatialBound lowerBound = box == null ? null : box.min();
+ GeospatialBound upperBound = box == null ? null : box.max();
+ Integer avgValueSizeInBytes =
+ valueCount > 0 ? Math.toIntExact(totalValueSizeInBytes / valueCount) : null;
+ return new GeometryFieldMetrics(
+ id, valueCount, lowerBound, upperBound, geometryType, avgValueSizeInBytes);
+ }
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/TestGeometryFieldMetrics.java b/core/src/test/java/org/apache/iceberg/TestGeometryFieldMetrics.java
new file mode 100644
index 0000000000..953835b62e
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/TestGeometryFieldMetrics.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import org.apache.iceberg.geospatial.GeospatialBound;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.Test;
+
+public class TestGeometryFieldMetrics {
+
+ private static final Types.GeometryType GEOMETRY = Types.GeometryType.crs84();
+
+ @Test
+ public void testBoundsAcrossMultiplePoints() {
+ GeometryFieldMetrics.Builder builder = new GeometryFieldMetrics.Builder(2, GEOMETRY);
+ builder.addValue(wkbPoint(30, 10));
+ builder.addValue(wkbPoint(-5, 40));
+
+ FieldMetrics<GeospatialBound> metrics = builder.build();
+
+ assertThat(metrics.id()).isEqualTo(2);
+ assertThat(metrics.valueCount()).isEqualTo(2L);
+ // null accounting is reconciled by the optional-field writer, not here
+ assertThat(metrics.nullValueCount()).isEqualTo(0L);
+ // geometry has no NaN metric; -1 suppresses the entry downstream
+ assertThat(metrics.nanValueCount()).isEqualTo(-1L);
+ assertThat(metrics.originalType()).isEqualTo(GEOMETRY);
+ assertThat(metrics.lowerBound()).isEqualTo(GeospatialBound.createXY(-5, 10));
+ assertThat(metrics.upperBound()).isEqualTo(GeospatialBound.createXY(30, 40));
+ // both WKB points are 21 bytes, so the average WKB size is 21
+ assertThat(metrics.avgValueSizeInBytes()).isEqualTo(21);
+ }
+
+ @Test
+ public void testSinglePointBoundsAreThatPoint() {
+ GeometryFieldMetrics.Builder builder = new GeometryFieldMetrics.Builder(2, GEOMETRY);
+ builder.addValue(wkbPoint(12, 34));
+
+ FieldMetrics<GeospatialBound> metrics = builder.build();
+
+ assertThat(metrics.valueCount()).isEqualTo(1L);
+ assertThat(metrics.lowerBound()).isEqualTo(GeospatialBound.createXY(12, 34));
+ assertThat(metrics.upperBound()).isEqualTo(GeospatialBound.createXY(12, 34));
+ assertThat(metrics.avgValueSizeInBytes()).isEqualTo(21);
+ }
+
+ @Test
+ public void testEmptyGeometryProducesNoBounds() {
+ // POINT EMPTY (NaN coordinates) still counts as a value but contributes no coordinate.
+ GeometryFieldMetrics.Builder builder = new GeometryFieldMetrics.Builder(2, GEOMETRY);
+ builder.addValue(wkbPoint(Double.NaN, Double.NaN));
+
+ FieldMetrics<GeospatialBound> metrics = builder.build();
+
+ assertThat(metrics.valueCount()).isEqualTo(1L);
+ assertThat(metrics.lowerBound()).isNull();
+ assertThat(metrics.upperBound()).isNull();
+ // the value has no bounds but still has a size, so the average is still reported
+ assertThat(metrics.avgValueSizeInBytes()).isEqualTo(21);
+ }
+
+ @Test
+ public void testNoValuesProducesNoBounds() {
+ FieldMetrics<GeospatialBound> metrics = new GeometryFieldMetrics.Builder(2, GEOMETRY).build();
+
+ assertThat(metrics.valueCount()).isEqualTo(0L);
+ assertThat(metrics.lowerBound()).isNull();
+ assertThat(metrics.upperBound()).isNull();
+ assertThat(metrics.avgValueSizeInBytes()).isNull();
+ }
+
+ private static ByteBuffer wkbPoint(double xCoord, double yCoord) {
+ return ByteBuffer.allocate(21)
+ .order(ByteOrder.LITTLE_ENDIAN)
+ .put((byte) 1) // little-endian
+ .putInt(1) // WKB geometry type: Point
+ .putDouble(xCoord)
+ .putDouble(yCoord)
+ .flip();
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/TestMetrics.java b/core/src/test/java/org/apache/iceberg/TestMetrics.java
index 3c68b0e39d..9b25825316 100644
--- a/core/src/test/java/org/apache/iceberg/TestMetrics.java
+++ b/core/src/test/java/org/apache/iceberg/TestMetrics.java
@@ -37,6 +37,7 @@ import java.util.List;
import java.util.Map;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.data.Record;
+import org.apache.iceberg.geospatial.GeospatialBound;
import org.apache.iceberg.io.InputFile;
import org.apache.iceberg.io.OutputFile;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
@@ -278,15 +279,22 @@ public abstract class TestMetrics {
first.setField("geog", geog);
Record second = GenericRecord.create(schema);
second.setField("id", 2L);
- // both geo columns are left null
+ second.setField("geom", wkbPoint(-5, 40));
+ // geog on the second row is left null
Metrics metrics = getMetrics(schema, first, second);
assertThat(metrics.recordCount()).isEqualTo(2L);
- // geometry and geography keep value/null counts but no bounds: lexicographic WKB min/max is not
- // meaningful, so bounds are intentionally skipped (spatial bounds are a separate follow-up).
- assertCounts(2, 2L, 1L, metrics);
- assertBounds(2, Types.GeometryType.crs84(), null, null, metrics);
+ // geometry bounds are the XY bounding box of all values: min corner (-5, 10), max corner
+ // (30, 40). The null geography row leaves geography with value/null counts but no bounds
+ // (geography bounds are a separate follow-up).
+ assertCounts(2, 2L, 0L, metrics);
+ assertBounds(
+ 2,
+ Types.GeometryType.crs84(),
+ GeospatialBound.createXY(-5, 10),
+ GeospatialBound.createXY(30, 40),
+ metrics);
assertCounts(3, 2L, 1L, metrics);
assertBounds(3, Types.GeographyType.crs84(), null, null, metrics);
assertThat(metrics.avgValueSizes())
diff --git a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java
index 78b924db5b..d14c05e0b2 100644
--- a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java
+++ b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java
@@ -268,8 +268,11 @@ abstract class BaseParquetWriter<T> {
@Override
public Optional<ParquetValueWriter<?>> visit(
LogicalTypeAnnotation.GeometryLogicalTypeAnnotation geometryType) {
- // geometry values are pure WKB stored in a BINARY column
- return Optional.of(ParquetValueWriters.geospatial(desc));
+ // geometry values are pure WKB stored in a BINARY column; the writer also scans the
+ // coordinates to produce a bounding box (the Parquet footer cannot). The concrete CRS is
+ // immaterial here: ParquetMetrics rewrites the bound's type from the table schema before
+ // serialization, and the serialization is keyed on the type id, which ignores the CRS.
+ return Optional.of(ParquetValueWriters.geometry(desc, Types.GeometryType.crs84()));
}
@Override
diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueWriters.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueWriters.java
index 352e01d3f4..89992ecfa0 100644
--- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueWriters.java
+++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueWriters.java
@@ -36,6 +36,7 @@ import org.apache.avro.util.Utf8;
import org.apache.iceberg.DoubleFieldMetrics;
import org.apache.iceberg.FieldMetrics;
import org.apache.iceberg.FloatFieldMetrics;
+import org.apache.iceberg.GeometryFieldMetrics;
import org.apache.iceberg.StructLike;
import org.apache.iceberg.ValueSizeFieldMetrics;
import org.apache.iceberg.deletes.PositionDelete;
@@ -125,6 +126,11 @@ public class ParquetValueWriters {
return new GeospatialWriter(desc);
}
+ public static PrimitiveWriter<ByteBuffer> geometry(
+ ColumnDescriptor desc, org.apache.iceberg.types.Type geometryType) {
+ return new GeometryWriter(desc, geometryType);
+ }
+
public static PrimitiveWriter<ByteBuffer> fixedBuffers(ColumnDescriptor desc) {
return new FixedBufferWriter(desc);
}
@@ -362,6 +368,30 @@ public class ParquetValueWriters {
}
}
+ private static class GeometryWriter extends PrimitiveWriter<ByteBuffer> {
+ private final GeometryFieldMetrics.Builder metricsBuilder;
+
+ private GeometryWriter(ColumnDescriptor desc, org.apache.iceberg.types.Type geometryType) {
+ super(desc);
+ this.metricsBuilder =
+ new GeometryFieldMetrics.Builder(
+ desc.getPrimitiveType().getId().intValue(), geometryType);
+ }
+
+ @Override
+ public void write(int repetitionLevel, ByteBuffer buffer) {
+ // Accumulate the bounding box before writing, so it reads the buffer's coordinates while the
+ // position is intact (the scanner reads a duplicate and leaves this buffer untouched).
+ metricsBuilder.addValue(buffer);
+ column.writeBinary(repetitionLevel, Binary.fromReusedByteBuffer(buffer));
+ }
+
+ @Override
+ public Stream<FieldMetrics<?>> metrics() {
+ return Stream.of(metricsBuilder.build());
+ }
+ }
+
private static class FixedBufferWriter extends PrimitiveWriter<ByteBuffer> {
private final int length;
diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetValueWriters.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetValueWriters.java
index 33c4044a0a..d5c968b117 100644
--- a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetValueWriters.java
+++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetValueWriters.java
@@ -60,4 +60,31 @@ class TestParquetValueWriters {
assertThat(metrics.nullValueCount()).isEqualTo(1);
assertThat(metrics.avgValueSizeInBytes()).isEqualTo(31);
}
+
+ @Test
+ void geometryValueSizeMetricsExcludeNulls() {
+ Schema schema = new Schema(optional(2, "geom", Types.GeometryType.crs84()));
+ MessageType parquetSchema = ParquetSchemaUtil.convert(schema, "table");
+ Type parquetType = parquetSchema.getType("geom");
+ ColumnDescriptor desc = parquetSchema.getColumnDescription(new String[] {"geom"});
+ ParquetValueWriter<ByteBuffer> writer =
+ ParquetValueWriters.option(
+ parquetType,
+ parquetSchema.getMaxDefinitionLevel(new String[] {"geom"}),
+ ParquetValueWriters.geometry(desc, Types.GeometryType.crs84()));
+
+ ColumnWriteStore columnStore = mock(ColumnWriteStore.class);
+ when(columnStore.getColumnWriter(desc)).thenReturn(mock(ColumnWriter.class));
+ writer.setColumnStore(columnStore);
+ writer.write(0, ByteBuffer.allocate(21));
+ writer.write(0, ByteBuffer.allocate(42));
+ writer.write(0, null);
+
+ // the geometry writer adds bounds but must keep the same average WKB size metric as the
+ // counts-only geospatial writer: the average is over the two non-null values, (21 + 42) / 2
+ FieldMetrics<?> metrics = writer.metrics().findFirst().orElseThrow();
+ assertThat(metrics.valueCount()).isEqualTo(3);
+ assertThat(metrics.nullValueCount()).isEqualTo(1);
+ assertThat(metrics.avgValueSizeInBytes()).isEqualTo(31);
+ }
}