You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by dw...@apache.org on 2026/09/11 20:25:11 UTC
(iceberg) branch main updated: Parquet: parse row-group size as long (#18055)
This is an automated email from the ASF dual-hosted git repository.
danielcweeks pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 4a0dc3f26d Parquet: parse row-group size as long (#18055)
4a0dc3f26d is described below
commit 4a0dc3f26d93e64e5b4f036cc37e3d2c81ee7cde
Author: Pritam <28...@users.noreply.github.com>
AuthorDate: Fri Sep 11 21:25:03 2026 +0100
Parquet: parse row-group size as long (#18055)
* Parquet: parse row-group size as long
---
.../java/org/apache/iceberg/parquet/Parquet.java | 18 ++++----
.../org/apache/iceberg/parquet/TestParquet.java | 50 ++++++++++++++++++++++
2 files changed, 59 insertions(+), 9 deletions(-)
diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java b/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java
index 1fff5baa95..d5a036e1f1 100644
--- a/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java
+++ b/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java
@@ -400,7 +400,7 @@ public class Parquet {
// Map Iceberg properties to pass down to the Parquet writer
Context context = createContextFunc.apply(config);
- int rowGroupSize = context.rowGroupSize();
+ long rowGroupSize = context.rowGroupSize();
int pageSize = context.pageSize();
int pageRowLimit = context.pageRowLimit();
int dictionaryPageSize = context.dictionaryPageSize();
@@ -517,7 +517,7 @@ public class Parquet {
.setWriteSupport(getWriteSupport(type))
.withCompressionCodec(codec)
.withWriteMode(writeMode)
- .withRowGroupSize((long) rowGroupSize)
+ .withRowGroupSize(rowGroupSize)
.withPageSize(pageSize)
.withPageRowCountLimit(pageRowLimit)
.withDictionaryEncoding(dictionaryEnabled)
@@ -542,7 +542,7 @@ public class Parquet {
}
static class Context {
- private final int rowGroupSize;
+ private final long rowGroupSize;
private final int pageSize;
private final int pageRowLimit;
private final int dictionaryPageSize;
@@ -562,7 +562,7 @@ public class Parquet {
private final boolean trackUncompressedRowGroupSize;
private Context(
- int rowGroupSize,
+ long rowGroupSize,
int pageSize,
int pageRowLimit,
int dictionaryPageSize,
@@ -601,8 +601,8 @@ public class Parquet {
}
static Context dataContext(Map<String, String> config) {
- int rowGroupSize =
- PropertyUtil.propertyAsInt(
+ long rowGroupSize =
+ PropertyUtil.propertyAsLong(
config, PARQUET_ROW_GROUP_SIZE_BYTES, PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT);
Preconditions.checkArgument(rowGroupSize > 0, "Row group size must be > 0");
@@ -711,8 +711,8 @@ public class Parquet {
// default delete config using data config
Context dataContext = dataContext(config);
- int rowGroupSize =
- PropertyUtil.propertyAsInt(
+ long rowGroupSize =
+ PropertyUtil.propertyAsLong(
config, DELETE_PARQUET_ROW_GROUP_SIZE_BYTES, dataContext.rowGroupSize());
Preconditions.checkArgument(rowGroupSize > 0, "Row group size must be > 0");
@@ -804,7 +804,7 @@ public class Parquet {
}
}
- int rowGroupSize() {
+ long rowGroupSize() {
return rowGroupSize;
}
diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java
index 5ab4afba6b..f5f84ea853 100644
--- a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java
+++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java
@@ -20,6 +20,7 @@ package org.apache.iceberg.parquet;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.iceberg.Files.localInput;
+import static org.apache.iceberg.TableProperties.DELETE_PARQUET_ROW_GROUP_SIZE_BYTES;
import static org.apache.iceberg.TableProperties.PARQUET_BLOOM_FILTER_ADAPTIVE_ENABLED;
import static org.apache.iceberg.TableProperties.PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX;
import static org.apache.iceberg.TableProperties.PARQUET_BLOOM_FILTER_MAX_BYTES;
@@ -55,10 +56,12 @@ import org.apache.avro.generic.GenericRecordBuilder;
import org.apache.iceberg.Files;
import org.apache.iceberg.Metrics;
import org.apache.iceberg.MetricsConfig;
+import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.avro.AvroSchemaUtil;
import org.apache.iceberg.data.Record;
import org.apache.iceberg.data.parquet.GenericParquetWriter;
+import org.apache.iceberg.deletes.EqualityDeleteWriter;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.io.FileAppender;
@@ -111,6 +114,53 @@ public class TestParquet {
}
}
+ @Test
+ public void rowGroupSizeLargerThanIntegerMax() throws IOException {
+ Schema schema = new Schema(optional(1, "intCol", IntegerType.get()));
+ org.apache.avro.Schema avroSchema = AvroSchemaUtil.convert(schema.asStruct());
+ GenericData.Record record = new GenericData.Record(avroSchema);
+ record.put("intCol", 1);
+
+ // Values above Integer.MAX_VALUE used to fail in PropertyUtil.propertyAsInt.
+ File file = createTempFile(temp);
+ write(
+ file,
+ schema,
+ ImmutableMap.of(PARQUET_ROW_GROUP_SIZE_BYTES, Long.toString(4L * 1024 * 1024 * 1024)),
+ ParquetAvroWriter::buildWriter,
+ record);
+
+ try (ParquetFileReader reader = ParquetFileReader.open(ParquetIO.file(localInput(file)))) {
+ assertThat(reader.getRowGroups()).hasSize(1);
+ }
+ }
+
+ @Test
+ public void deleteRowGroupSizeLargerThanIntegerMax() throws IOException {
+ Schema schema = new Schema(required(1, "id", Types.LongType.get()));
+ Record record = org.apache.iceberg.data.GenericRecord.create(schema);
+ record.setField("id", 1L);
+
+ File file = createTempFile(temp);
+ EqualityDeleteWriter<Record> deleteWriter =
+ Parquet.writeDeletes(Files.localOutput(file))
+ .createWriterFunc(GenericParquetWriter::create)
+ .set(DELETE_PARQUET_ROW_GROUP_SIZE_BYTES, Long.toString(4L * 1024 * 1024 * 1024))
+ .overwrite()
+ .rowSchema(schema)
+ .withSpec(PartitionSpec.unpartitioned())
+ .equalityFieldIds(1)
+ .buildEqualityWriter();
+
+ try (EqualityDeleteWriter<Record> writer = deleteWriter) {
+ writer.write(record);
+ }
+
+ try (ParquetFileReader reader = ParquetFileReader.open(ParquetIO.file(localInput(file)))) {
+ assertThat(reader.getRowGroups()).hasSize(1);
+ }
+ }
+
@Test
public void testRowGroupSizeConfigurableWithWriter() throws IOException {
// Explicit writer function supports PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT