You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by pv...@apache.org on 2025/02/28 13:21:58 UTC
(iceberg) branch main updated: Data: Add partition stats writer and reader (#11216)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new e230f5d79d Data: Add partition stats writer and reader (#11216)
e230f5d79d is described below
commit e230f5d79d82a50439029db5c73f8b59497b2e9f
Author: Ajantha Bhat <aj...@gmail.com>
AuthorDate: Fri Feb 28 18:51:53 2025 +0530
Data: Add partition stats writer and reader (#11216)
---
.../apache/iceberg/UpdatePartitionStatistics.java | 2 +-
.../org/apache/iceberg/SetPartitionStatistics.java | 6 +-
.../org/apache/iceberg/TestRowLineageMetadata.java | 5 +-
.../test/java/org/apache/iceberg/TestTables.java | 63 ++-
.../apache/iceberg/data/PartitionStatsHandler.java | 284 ++++++++++
.../iceberg/data/TestPartitionStatsHandler.java | 598 +++++++++++++++++++++
6 files changed, 920 insertions(+), 38 deletions(-)
diff --git a/api/src/main/java/org/apache/iceberg/UpdatePartitionStatistics.java b/api/src/main/java/org/apache/iceberg/UpdatePartitionStatistics.java
index 6590799d34..6156bdb271 100644
--- a/api/src/main/java/org/apache/iceberg/UpdatePartitionStatistics.java
+++ b/api/src/main/java/org/apache/iceberg/UpdatePartitionStatistics.java
@@ -24,7 +24,7 @@ import java.util.List;
public interface UpdatePartitionStatistics extends PendingUpdate<List<PartitionStatisticsFile>> {
/**
* Set the table's partition statistics file for given snapshot, replacing the previous partition
- * statistics file for the snapshot if any exists.
+ * statistics file for the snapshot if any exists. No-op if the provided file is null.
*
* @return this for method chaining
*/
diff --git a/core/src/main/java/org/apache/iceberg/SetPartitionStatistics.java b/core/src/main/java/org/apache/iceberg/SetPartitionStatistics.java
index aceb72e6a6..6ceb4f59d8 100644
--- a/core/src/main/java/org/apache/iceberg/SetPartitionStatistics.java
+++ b/core/src/main/java/org/apache/iceberg/SetPartitionStatistics.java
@@ -21,7 +21,6 @@ package org.apache.iceberg;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
@@ -36,7 +35,10 @@ public class SetPartitionStatistics implements UpdatePartitionStatistics {
@Override
public UpdatePartitionStatistics setPartitionStatistics(PartitionStatisticsFile file) {
- Preconditions.checkArgument(null != file, "partition statistics file must not be null");
+ if (file == null) {
+ return this;
+ }
+
statsToSet.put(file.snapshotId(), file);
return this;
}
diff --git a/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java b/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java
index 3a72f65403..fd8e000789 100644
--- a/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java
+++ b/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java
@@ -316,8 +316,9 @@ public class TestRowLineageMetadata {
tableDir,
"test",
TEST_SCHEMA,
- ImmutableMap.of(TableProperties.ROW_LINEAGE, "true"),
- formatVersion);
+ PartitionSpec.unpartitioned(),
+ formatVersion,
+ ImmutableMap.of(TableProperties.ROW_LINEAGE, "true"));
assertThat(table.ops().current().rowLineageEnabled()).isTrue();
}
diff --git a/core/src/test/java/org/apache/iceberg/TestTables.java b/core/src/test/java/org/apache/iceberg/TestTables.java
index 63d8a9476e..cf1480747c 100644
--- a/core/src/test/java/org/apache/iceberg/TestTables.java
+++ b/core/src/test/java/org/apache/iceberg/TestTables.java
@@ -53,23 +53,13 @@ public class TestTables {
}
public static TestTable create(
- File temp, String name, Schema schema, Map<String, String> properties, int formatVersion) {
- TestTableOperations ops = new TestTableOperations(name, temp);
- if (ops.current() != null) {
- throw new AlreadyExistsException("Table %s already exists at location: %s", name, temp);
- }
-
- ops.commit(
- null,
- newTableMetadata(
- schema,
- PartitionSpec.unpartitioned(),
- SortOrder.unsorted(),
- temp.toString(),
- properties,
- formatVersion));
-
- return new TestTable(ops);
+ File temp,
+ String name,
+ Schema schema,
+ PartitionSpec spec,
+ SortOrder sortOrder,
+ int formatVersion) {
+ return createTable(temp, name, schema, spec, formatVersion, ImmutableMap.of(), sortOrder, null);
}
public static TestTable create(
@@ -78,18 +68,10 @@ public class TestTables {
Schema schema,
PartitionSpec spec,
SortOrder sortOrder,
- int formatVersion) {
- TestTableOperations ops = new TestTableOperations(name, temp);
- if (ops.current() != null) {
- throw new AlreadyExistsException("Table %s already exists at location: %s", name, temp);
- }
-
- ops.commit(
- null,
- newTableMetadata(
- schema, spec, sortOrder, temp.toString(), ImmutableMap.of(), formatVersion));
-
- return new TestTable(ops);
+ int formatVersion,
+ MetricsReporter reporter) {
+ return createTable(
+ temp, name, schema, spec, formatVersion, ImmutableMap.of(), sortOrder, reporter);
}
public static TestTable create(
@@ -97,8 +79,20 @@ public class TestTables {
String name,
Schema schema,
PartitionSpec spec,
- SortOrder sortOrder,
int formatVersion,
+ Map<String, String> properties) {
+ return createTable(
+ temp, name, schema, spec, formatVersion, properties, SortOrder.unsorted(), null);
+ }
+
+ private static TestTable createTable(
+ File temp,
+ String name,
+ Schema schema,
+ PartitionSpec spec,
+ int formatVersion,
+ Map<String, String> properties,
+ SortOrder sortOrder,
MetricsReporter reporter) {
TestTableOperations ops = new TestTableOperations(name, temp);
if (ops.current() != null) {
@@ -107,10 +101,13 @@ public class TestTables {
ops.commit(
null,
- newTableMetadata(
- schema, spec, sortOrder, temp.toString(), ImmutableMap.of(), formatVersion));
+ newTableMetadata(schema, spec, sortOrder, temp.toString(), properties, formatVersion));
- return new TestTable(ops, reporter);
+ if (reporter != null) {
+ return new TestTable(ops, reporter);
+ } else {
+ return new TestTable(ops);
+ }
}
public static Transaction beginCreate(File temp, String name, Schema schema, PartitionSpec spec) {
diff --git a/data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java b/data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java
new file mode 100644
index 0000000000..a194a08255
--- /dev/null
+++ b/data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a combined partition type which considers
+ * all specs in a table.
+ *
+ * @param unifiedPartitionType unified partition schema type. Could be calculated by {@link
+ * Partitioning#partitionType(Table)}.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType unifiedPartitionType) {
+ Preconditions.checkState(!unifiedPartitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), unifiedPartitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, SnapshotRef.MAIN_BRANCH);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null || branch.equals(SnapshotRef.MAIN_BRANCH),
+ "Couldn't find the snapshot for the branch %s",
+ branch);
+ return null;
+ }
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterable<PartitionStats> records)
+ throws IOException {
+ FileFormat fileFormat =
+ FileFormat.fromString(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+
+ OutputFile outputFile = newPartitionStatsFile(table, fileFormat, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile, fileFormat)) {
+ records.iterator().forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static OutputFile newPartitionStatsFile(
+ Table table, FileFormat fileFormat, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(
+ Locale.ROOT, "partition-stats-%d-%s", snapshotId, UUID.randomUUID()))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(
+ Schema dataSchema, OutputFile outputFile, FileFormat fileFormat) throws IOException {
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(InternalWriter::create)
+ .withSpec(PartitionSpec.unpartitioned())
+ .build();
+ case AVRO:
+ return Avro.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(org.apache.iceberg.avro.InternalWriter::create)
+ .withSpec(PartitionSpec.unpartitioned())
+ .build();
+ case ORC:
+ // Internal writers are not supported for ORC yet.
+ default:
+ throw new UnsupportedOperationException("Unsupported file format:" + fileFormat.name());
+ }
+ }
+
+ private static CloseableIterable<StructLike> dataReader(Schema schema, InputFile inputFile) {
+ FileFormat fileFormat = FileFormat.fromFileName(inputFile.location());
+ Preconditions.checkArgument(
+ fileFormat != null, "Unable to determine format of file: %s", inputFile.location());
+
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.read(inputFile)
+ .project(schema)
+ .createReaderFunc(
+ fileSchema ->
+ org.apache.iceberg.data.parquet.InternalReader.create(schema, fileSchema))
+ .build();
+ case AVRO:
+ return Avro.read(inputFile)
+ .project(schema)
+ .createReaderFunc(fileSchema -> InternalReader.create(schema))
+ .build();
+ case ORC:
+ // Internal readers are not supported for ORC yet.
+ default:
+ throw new UnsupportedOperationException("Unsupported file format:" + fileFormat.name());
+ }
+ }
+
+ private static PartitionStats recordToPartitionStats(StructLike record) {
+ PartitionStats stats =
+ new PartitionStats(
+ record.get(Column.PARTITION.id(), StructLike.class),
+ record.get(Column.SPEC_ID.id(), Integer.class));
+ stats.set(Column.DATA_RECORD_COUNT.id(), record.get(Column.DATA_RECORD_COUNT.id(), Long.class));
+ stats.set(Column.DATA_FILE_COUNT.id(), record.get(Column.DATA_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(),
+ record.get(Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(), Long.class));
+ stats.set(
+ Column.POSITION_DELETE_RECORD_COUNT.id(),
+ record.get(Column.POSITION_DELETE_RECORD_COUNT.id(), Long.class));
+ stats.set(
+ Column.POSITION_DELETE_FILE_COUNT.id(),
+ record.get(Column.POSITION_DELETE_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.EQUALITY_DELETE_RECORD_COUNT.id(),
+ record.get(Column.EQUALITY_DELETE_RECORD_COUNT.id(), Long.class));
+ stats.set(
+ Column.EQUALITY_DELETE_FILE_COUNT.id(),
+ record.get(Column.EQUALITY_DELETE_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.TOTAL_RECORD_COUNT.id(), record.get(Column.TOTAL_RECORD_COUNT.id(), Long.class));
+ stats.set(Column.LAST_UPDATED_AT.id(), record.get(Column.LAST_UPDATED_AT.id(), Long.class));
+ stats.set(
+ Column.LAST_UPDATED_SNAPSHOT_ID.id(),
+ record.get(Column.LAST_UPDATED_SNAPSHOT_ID.id(), Long.class));
+ return stats;
+ }
+}
diff --git a/data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java b/data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java
new file mode 100644
index 0000000000..8256f80d9b
--- /dev/null
+++ b/data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java
@@ -0,0 +1,598 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.data.PartitionStatsHandler.Column;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionData;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assumptions;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestPartitionStatsHandler {
+ private static final Schema SCHEMA =
+ new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get()));
+
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("c2").identity("c3").build();
+
+ @TempDir public File temp;
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameters(name = "fileFormat = {0}")
+ public static List<Object> parameters() {
+ return Arrays.asList(FileFormat.PARQUET, FileFormat.ORC, FileFormat.AVRO);
+ }
+
+ @Parameter private FileFormat format;
+
+ @Test
+ public void testPartitionStatsOnEmptyTable() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_table"), "empty_table", SCHEMA, SPEC, 2);
+ assertThat(PartitionStatsHandler.computeAndWriteStatsFile(testTable)).isNull();
+ }
+
+ @Test
+ public void testPartitionStatsOnEmptyBranch() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_branch"), "empty_branch", SCHEMA, SPEC, 2);
+ testTable.manageSnapshots().createBranch("b1").commit();
+ assertThat(PartitionStatsHandler.computeAndWriteStatsFile(testTable, "b1")).isNull();
+ }
+
+ @Test
+ public void testPartitionStatsOnInvalidSnapshot() throws Exception {
+ Table testTable =
+ TestTables.create(tempDir("invalid_snapshot"), "invalid_snapshot", SCHEMA, SPEC, 2);
+ assertThatThrownBy(
+ () -> PartitionStatsHandler.computeAndWriteStatsFile(testTable, "INVALID_BRANCH"))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Couldn't find the snapshot for the branch INVALID_BRANCH");
+ }
+
+ @Test
+ public void testPartitionStatsOnUnPartitionedTable() throws Exception {
+ Table testTable =
+ TestTables.create(
+ tempDir("unpartitioned_table"),
+ "unpartitioned_table",
+ SCHEMA,
+ PartitionSpec.unpartitioned(),
+ 2);
+
+ List<Record> records = prepareRecords(testTable.schema());
+ DataFile dataFile = FileHelpers.writeDataFile(testTable, outputFile(), records);
+ testTable.newAppend().appendFile(dataFile).commit();
+
+ assertThatThrownBy(() -> PartitionStatsHandler.computeAndWriteStatsFile(testTable))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("table must be partitioned");
+ }
+
+ @Test
+ public void testAllDatatypePartitionWriting() throws Exception {
+ Schema schema =
+ new Schema(
+ required(100, "id", Types.LongType.get()),
+ optional(101, "data", Types.StringType.get()),
+ required(102, "b", Types.BooleanType.get()),
+ optional(103, "i", Types.IntegerType.get()),
+ required(104, "l", Types.LongType.get()),
+ optional(105, "f", Types.FloatType.get()),
+ required(106, "d", Types.DoubleType.get()),
+ optional(107, "date", Types.DateType.get()),
+ required(108, "ts", Types.TimestampType.withoutZone()),
+ required(110, "s", Types.StringType.get()),
+ required(111, "uuid", Types.UUIDType.get()),
+ required(112, "fixed", Types.FixedType.ofLength(7)),
+ optional(113, "bytes", Types.BinaryType.get()),
+ required(114, "dec_9_0", Types.DecimalType.of(9, 0)),
+ required(115, "dec_11_2", Types.DecimalType.of(11, 2)),
+ required(116, "dec_38_10", Types.DecimalType.of(38, 10)), // maximum precision
+ required(117, "time", Types.TimeType.get()));
+
+ PartitionSpec spec =
+ PartitionSpec.builderFor(schema)
+ .identity("b")
+ .identity("i")
+ .identity("l")
+ .identity("f")
+ .identity("d")
+ .identity("date")
+ .identity("ts")
+ .identity("s")
+ .identity("uuid")
+ .identity("fixed")
+ .identity("bytes")
+ .identity("dec_9_0")
+ .identity("dec_11_2")
+ .identity("dec_38_10")
+ .identity("time")
+ .build();
+
+ Table testTable =
+ TestTables.create(
+ tempDir("test_all_type"), "test_all_type", schema, spec, SortOrder.unsorted(), 2);
+
+ Types.StructType partitionSchema = Partitioning.partitionType(testTable);
+ Schema dataSchema = PartitionStatsHandler.schema(partitionSchema);
+
+ PartitionData partitionData =
+ new PartitionData(dataSchema.findField(Column.PARTITION.name()).type().asStructType());
+ partitionData.set(0, true);
+ partitionData.set(1, 42);
+ partitionData.set(2, 42L);
+ partitionData.set(3, 3.14f);
+ partitionData.set(4, 3.141592653589793);
+ partitionData.set(5, Literal.of("2022-01-01").to(Types.DateType.get()).value());
+ partitionData.set(
+ 6, Literal.of("2017-12-01T10:12:55.038194").to(Types.TimestampType.withoutZone()).value());
+ partitionData.set(7, "string");
+ partitionData.set(8, UUID.randomUUID());
+ partitionData.set(9, ByteBuffer.wrap(new byte[] {0, 1, 2, 3, 4, 5, 6}));
+ partitionData.set(10, ByteBuffer.wrap(new byte[] {1, 2, 3}));
+ partitionData.set(11, new BigDecimal("123456789"));
+ partitionData.set(12, new BigDecimal("1234567.89"));
+ partitionData.set(13, new BigDecimal("12345678901234567890.1234567890"));
+ partitionData.set(14, Literal.of("10:10:10").to(Types.TimeType.get()).value());
+
+ PartitionStats partitionStats = new PartitionStats(partitionData, RANDOM.nextInt(10));
+ partitionStats.set(Column.DATA_RECORD_COUNT.id(), RANDOM.nextLong());
+ partitionStats.set(Column.DATA_FILE_COUNT.id(), RANDOM.nextInt());
+ partitionStats.set(Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(), 1024L * RANDOM.nextInt(20));
+ List<PartitionStats> expected = Collections.singletonList(partitionStats);
+ PartitionStatisticsFile statisticsFile =
+ PartitionStatsHandler.writePartitionStatsFile(testTable, 42L, dataSchema, expected);
+
+ List<PartitionStats> written;
+ try (CloseableIterable<PartitionStats> recordIterator =
+ PartitionStatsHandler.readPartitionStatsFile(
+ dataSchema, Files.localInput(statisticsFile.path()))) {
+ written = Lists.newArrayList(recordIterator);
+ }
+
+ assertThat(written).hasSize(expected.size());
+ Comparator<StructLike> comparator = Comparators.forType(partitionSchema);
+ for (int i = 0; i < written.size(); i++) {
+ assertThat(isEqual(comparator, written.get(i), expected.get(i))).isTrue();
+ }
+ }
+
+ @Test
+ public void testOptionalFieldsWriting() throws Exception {
+ PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).identity("c1").build();
+ Table testTable =
+ TestTables.create(
+ tempDir("test_partition_stats_optional"),
+ "test_partition_stats_optional",
+ SCHEMA,
+ spec,
+ SortOrder.unsorted(),
+ 2);
+
+ Types.StructType partitionSchema = Partitioning.partitionType(testTable);
+ Schema dataSchema = PartitionStatsHandler.schema(partitionSchema);
+
+ ImmutableList.Builder<PartitionStats> partitionListBuilder = ImmutableList.builder();
+ for (int i = 0; i < 5; i++) {
+ PartitionData partitionData =
+ new PartitionData(dataSchema.findField(Column.PARTITION.name()).type().asStructType());
+ partitionData.set(0, RANDOM.nextInt());
+
+ PartitionStats stats = new PartitionStats(partitionData, RANDOM.nextInt(10));
+ stats.set(Column.PARTITION.ordinal(), partitionData);
+ stats.set(Column.DATA_RECORD_COUNT.ordinal(), RANDOM.nextLong());
+ stats.set(Column.DATA_FILE_COUNT.ordinal(), RANDOM.nextInt());
+ stats.set(Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.ordinal(), 1024L * RANDOM.nextInt(20));
+ stats.set(Column.POSITION_DELETE_RECORD_COUNT.ordinal(), null);
+ stats.set(Column.POSITION_DELETE_FILE_COUNT.ordinal(), null);
+ stats.set(Column.EQUALITY_DELETE_RECORD_COUNT.ordinal(), null);
+ stats.set(Column.EQUALITY_DELETE_FILE_COUNT.ordinal(), null);
+ stats.set(Column.TOTAL_RECORD_COUNT.ordinal(), null);
+ stats.set(Column.LAST_UPDATED_AT.ordinal(), null);
+ stats.set(Column.LAST_UPDATED_SNAPSHOT_ID.ordinal(), null);
+
+ partitionListBuilder.add(stats);
+ }
+
+ List<PartitionStats> expected = partitionListBuilder.build();
+
+ assertThat(expected.get(0))
+ .extracting(
+ PartitionStats::positionDeleteRecordCount,
+ PartitionStats::positionDeleteFileCount,
+ PartitionStats::equalityDeleteRecordCount,
+ PartitionStats::equalityDeleteFileCount,
+ PartitionStats::totalRecordCount,
+ PartitionStats::lastUpdatedAt,
+ PartitionStats::lastUpdatedSnapshotId)
+ .isEqualTo(
+ Arrays.asList(
+ 0L, 0, 0L, 0, 0L, null, null)); // null counters must be initialized to zero.
+
+ PartitionStatisticsFile statisticsFile =
+ PartitionStatsHandler.writePartitionStatsFile(testTable, 42L, dataSchema, expected);
+
+ List<PartitionStats> written;
+ try (CloseableIterable<PartitionStats> recordIterator =
+ PartitionStatsHandler.readPartitionStatsFile(
+ dataSchema, Files.localInput(statisticsFile.path()))) {
+ written = Lists.newArrayList(recordIterator);
+ }
+
+ assertThat(written).hasSize(expected.size());
+ Comparator<StructLike> comparator = Comparators.forType(partitionSchema);
+ for (int i = 0; i < written.size(); i++) {
+ assertThat(isEqual(comparator, written.get(i), expected.get(i))).isTrue();
+ }
+ }
+
+ @SuppressWarnings("checkstyle:MethodLength")
+ @TestTemplate // Tests for all the table formats (PARQUET, ORC, AVRO)
+ public void testPartitionStats() throws Exception {
+ Assumptions.assumeThat(format)
+ .as("ORC internal readers and writers are not supported")
+ .isNotEqualTo(FileFormat.ORC);
+
+ Table testTable =
+ TestTables.create(
+ tempDir("partition_stats_" + format.name()),
+ "partition_stats_compute_" + format.name(),
+ SCHEMA,
+ SPEC,
+ 2,
+ ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()));
+
+ List<Record> records = prepareRecords(testTable.schema());
+ DataFile dataFile1 =
+ FileHelpers.writeDataFile(
+ testTable, outputFile(), TestHelpers.Row.of("foo", "A"), records.subList(0, 3));
+ DataFile dataFile2 =
+ FileHelpers.writeDataFile(
+ testTable, outputFile(), TestHelpers.Row.of("foo", "B"), records.subList(3, 4));
+ DataFile dataFile3 =
+ FileHelpers.writeDataFile(
+ testTable, outputFile(), TestHelpers.Row.of("bar", "A"), records.subList(4, 5));
+ DataFile dataFile4 =
+ FileHelpers.writeDataFile(
+ testTable, outputFile(), TestHelpers.Row.of("bar", "B"), records.subList(5, 7));
+
+ for (int i = 0; i < 3; i++) {
+ // insert same set of seven records thrice to have a new manifest files
+ testTable
+ .newAppend()
+ .appendFile(dataFile1)
+ .appendFile(dataFile2)
+ .appendFile(dataFile3)
+ .appendFile(dataFile4)
+ .commit();
+ }
+
+ Snapshot snapshot1 = testTable.currentSnapshot();
+ Schema recordSchema = PartitionStatsHandler.schema(Partitioning.partitionType(testTable));
+ Types.StructType partitionType =
+ recordSchema.findField(Column.PARTITION.name()).type().asStructType();
+ computeAndValidatePartitionStats(
+ testTable,
+ recordSchema,
+ Tuple.tuple(
+ partitionRecord(partitionType, "foo", "A"),
+ 0,
+ 9L,
+ 3,
+ 3 * dataFile1.fileSizeInBytes(),
+ 0L,
+ 0,
+ 0L,
+ 0,
+ 0L,
+ snapshot1.timestampMillis(),
+ snapshot1.snapshotId()),
+ Tuple.tuple(
+ partitionRecord(partitionType, "foo", "B"),
+ 0,
+ 3L,
+ 3,
+ 3 * dataFile2.fileSizeInBytes(),
+ 0L,
+ 0,
+ 0L,
+ 0,
+ 0L,
+ snapshot1.timestampMillis(),
+ snapshot1.snapshotId()),
+ Tuple.tuple(
+ partitionRecord(partitionType, "bar", "A"),
+ 0,
+ 3L,
+ 3,
+ 3 * dataFile3.fileSizeInBytes(),
+ 0L,
+ 0,
+ 0L,
+ 0,
+ 0L,
+ snapshot1.timestampMillis(),
+ snapshot1.snapshotId()),
+ Tuple.tuple(
+ partitionRecord(partitionType, "bar", "B"),
+ 0,
+ 6L,
+ 3,
+ 3 * dataFile4.fileSizeInBytes(),
+ 0L,
+ 0,
+ 0L,
+ 0,
+ 0L,
+ snapshot1.timestampMillis(),
+ snapshot1.snapshotId()));
+
+ DeleteFile posDeletes = commitPositionDeletes(testTable, dataFile1);
+ Snapshot snapshot2 = testTable.currentSnapshot();
+
+ DeleteFile eqDeletes = commitEqualityDeletes(testTable);
+ Snapshot snapshot3 = testTable.currentSnapshot();
+
+ recordSchema = PartitionStatsHandler.schema(Partitioning.partitionType(testTable));
+ partitionType = recordSchema.findField(Column.PARTITION.name()).type().asStructType();
+ computeAndValidatePartitionStats(
+ testTable,
+ recordSchema,
+ Tuple.tuple(
+ partitionRecord(partitionType, "foo", "A"),
+ 0,
+ 9L,
+ 3,
+ 3 * dataFile1.fileSizeInBytes(),
+ 0L,
+ 0,
+ eqDeletes.recordCount(),
+ 1,
+ 0L,
+ snapshot3.timestampMillis(),
+ snapshot3.snapshotId()),
+ Tuple.tuple(
+ partitionRecord(partitionType, "foo", "B"),
+ 0,
+ 3L,
+ 3,
+ 3 * dataFile2.fileSizeInBytes(),
+ 0L,
+ 0,
+ 0L,
+ 0,
+ 0L,
+ snapshot1.timestampMillis(),
+ snapshot1.snapshotId()),
+ Tuple.tuple(
+ partitionRecord(partitionType, "bar", "A"),
+ 0,
+ 3L,
+ 3,
+ 3 * dataFile3.fileSizeInBytes(),
+ posDeletes.recordCount(),
+ 1,
+ 0L,
+ 0,
+ 0L,
+ snapshot2.timestampMillis(),
+ snapshot2.snapshotId()),
+ Tuple.tuple(
+ partitionRecord(partitionType, "bar", "B"),
+ 0,
+ 6L,
+ 3,
+ 3 * dataFile4.fileSizeInBytes(),
+ 0L,
+ 0,
+ 0L,
+ 0,
+ 0L,
+ snapshot1.timestampMillis(),
+ snapshot1.snapshotId()));
+ }
+
+ private OutputFile outputFile() throws IOException {
+ return Files.localOutput(File.createTempFile("data", null, tempDir("stats")));
+ }
+
+ private static StructLike partitionRecord(
+ Types.StructType partitionType, String val1, String val2) {
+ GenericRecord record = GenericRecord.create(partitionType);
+ record.set(0, val1);
+ record.set(1, val2);
+ return record;
+ }
+
+ private static List<Record> prepareRecords(Schema schema) {
+ GenericRecord record = GenericRecord.create(schema);
+ List<Record> records = Lists.newArrayList();
+ // foo 4 records, bar 3 records
+ // foo, A -> 3 records
+ records.add(record.copy("c1", 0, "c2", "foo", "c3", "A"));
+ records.add(record.copy("c1", 1, "c2", "foo", "c3", "A"));
+ records.add(record.copy("c1", 2, "c2", "foo", "c3", "A"));
+ // foo, B -> 1 record
+ records.add(record.copy("c1", 3, "c2", "foo", "c3", "B"));
+ // bar, A -> 1 record
+ records.add(record.copy("c1", 4, "c2", "bar", "c3", "A"));
+ // bar, B -> 2 records
+ records.add(record.copy("c1", 5, "c2", "bar", "c3", "B"));
+ records.add(record.copy("c1", 6, "c2", "bar", "c3", "B"));
+ return records;
+ }
+
+ private static void computeAndValidatePartitionStats(
+ Table testTable, Schema recordSchema, Tuple... expectedValues) throws IOException {
+ // compute and commit partition stats file
+ Snapshot currentSnapshot = testTable.currentSnapshot();
+ PartitionStatisticsFile result = PartitionStatsHandler.computeAndWriteStatsFile(testTable);
+ testTable.updatePartitionStatistics().setPartitionStatistics(result).commit();
+ assertThat(result.snapshotId()).isEqualTo(currentSnapshot.snapshotId());
+
+ // read the partition entries from the stats file
+ List<PartitionStats> partitionStats;
+ try (CloseableIterable<PartitionStats> recordIterator =
+ PartitionStatsHandler.readPartitionStatsFile(
+ recordSchema, Files.localInput(result.path()))) {
+ partitionStats = Lists.newArrayList(recordIterator);
+ }
+
+ assertThat(partitionStats)
+ .extracting(
+ PartitionStats::partition,
+ PartitionStats::specId,
+ PartitionStats::dataRecordCount,
+ PartitionStats::dataFileCount,
+ PartitionStats::totalDataFileSizeInBytes,
+ PartitionStats::positionDeleteRecordCount,
+ PartitionStats::positionDeleteFileCount,
+ PartitionStats::equalityDeleteRecordCount,
+ PartitionStats::equalityDeleteFileCount,
+ PartitionStats::totalRecordCount,
+ PartitionStats::lastUpdatedAt,
+ PartitionStats::lastUpdatedSnapshotId)
+ .containsExactlyInAnyOrder(expectedValues);
+ }
+
+ private DeleteFile commitEqualityDeletes(Table testTable) throws IOException {
+ Schema deleteRowSchema = testTable.schema().select("c1");
+ Record dataDelete = GenericRecord.create(deleteRowSchema);
+ List<Record> dataDeletes =
+ Lists.newArrayList(dataDelete.copy("c1", 1), dataDelete.copy("c1", 2));
+
+ DeleteFile eqDeletes =
+ FileHelpers.writeDeleteFile(
+ testTable,
+ Files.localOutput(File.createTempFile("junit", null, tempDir("eq_delete"))),
+ TestHelpers.Row.of("foo", "A"),
+ dataDeletes,
+ deleteRowSchema);
+ testTable.newRowDelta().addDeletes(eqDeletes).commit();
+ return eqDeletes;
+ }
+
+ private DeleteFile commitPositionDeletes(Table testTable, DataFile dataFile1) throws IOException {
+ List<PositionDelete<?>> deletes = Lists.newArrayList();
+ for (long i = 0; i < 2; i++) {
+ deletes.add(
+ positionDelete(testTable.schema(), dataFile1.location(), i, (int) i, String.valueOf(i)));
+ }
+
+ DeleteFile posDeletes =
+ FileHelpers.writePosDeleteFile(
+ testTable,
+ Files.localOutput(File.createTempFile("junit", null, tempDir("pos_delete"))),
+ TestHelpers.Row.of("bar", "A"),
+ deletes);
+ testTable.newRowDelta().addDeletes(posDeletes).commit();
+ return posDeletes;
+ }
+
+ private static PositionDelete<GenericRecord> positionDelete(
+ Schema tableSchema, CharSequence path, Long position, Object... values) {
+ PositionDelete<GenericRecord> posDelete = PositionDelete.create();
+ GenericRecord nested = GenericRecord.create(tableSchema);
+ for (int i = 0; i < values.length; i++) {
+ nested.set(i, values[i]);
+ }
+
+ posDelete.set(path, position, nested);
+ return posDelete;
+ }
+
+ private File tempDir(String folderName) throws IOException {
+ return java.nio.file.Files.createTempDirectory(temp.toPath(), folderName).toFile();
+ }
+
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ private static boolean isEqual(
+ Comparator<StructLike> partitionComparator, PartitionStats stats1, PartitionStats stats2) {
+ if (stats1 == stats2) {
+ return true;
+ } else if (stats1 == null || stats2 == null) {
+ return false;
+ }
+
+ return partitionComparator.compare(stats1.partition(), stats2.partition()) == 0
+ && stats1.specId() == stats2.specId()
+ && stats1.dataRecordCount() == stats2.dataRecordCount()
+ && stats1.dataFileCount() == stats2.dataFileCount()
+ && stats1.totalDataFileSizeInBytes() == stats2.totalDataFileSizeInBytes()
+ && stats1.positionDeleteRecordCount() == stats2.positionDeleteRecordCount()
+ && stats1.positionDeleteFileCount() == stats2.positionDeleteFileCount()
+ && stats1.equalityDeleteRecordCount() == stats2.equalityDeleteRecordCount()
+ && stats1.equalityDeleteFileCount() == stats2.equalityDeleteFileCount()
+ && stats1.totalRecordCount() == stats2.totalRecordCount()
+ && Objects.equals(stats1.lastUpdatedAt(), stats2.lastUpdatedAt())
+ && Objects.equals(stats1.lastUpdatedSnapshotId(), stats2.lastUpdatedSnapshotId());
+ }
+}