You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@iceberg.apache.org by "ajantha-bhat (via GitHub)" <gi...@apache.org> on 2024/09/26 17:00:09 UTC
[PR] Data: Add partition stats writer and reader [iceberg]
ajantha-bhat opened a new pull request, #11216:
URL: https://github.com/apache/iceberg/pull/11216
(no comment)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1818891380
##########
data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java:
##########
@@ -0,0 +1,569 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.data.PartitionStatsHandler.Column;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.UUIDUtil;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestPartitionStatsHandler {
+ private static final Schema SCHEMA =
+ new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get()));
+
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("c2").identity("c3").build();
+
+ @TempDir public File temp;
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameters(name = "fileFormat = {0}")
+ public static List<Object> parameters() {
+ return Arrays.asList(FileFormat.PARQUET, FileFormat.ORC, FileFormat.AVRO);
+ }
+
+ @Parameter private FileFormat format;
+
+ @Test
+ public void testPartitionStatsOnEmptyTable() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_table"), "empty_table", SCHEMA, SPEC, 2);
+ assertThat(PartitionStatsHandler.computeAndWriteStatsFile(testTable)).isNull();
+ }
+
+ @Test
+ public void testPartitionStatsOnEmptyBranch() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_branch"), "empty_branch", SCHEMA, SPEC, 2);
+ testTable.manageSnapshots().createBranch("b1").commit();
+ PartitionStatisticsFile partitionStatisticsFile =
+ PartitionStatsHandler.computeAndWriteStatsFile(testTable, "b1");
+ // creates an empty stats file since the dummy snapshot exist
+ assertThat(partitionStatisticsFile.fileSizeInBytes()).isEqualTo(0L);
+ assertThat(partitionStatisticsFile.snapshotId())
+ .isEqualTo(testTable.refs().get("b1").snapshotId());
+ }
+
+ @Test
+ public void testPartitionStatsOnInvalidSnapshot() throws Exception {
+ Table testTable =
+ TestTables.create(tempDir("invalid_snapshot"), "invalid_snapshot", SCHEMA, SPEC, 2);
+ assertThatThrownBy(
+ () -> PartitionStatsHandler.computeAndWriteStatsFile(testTable, "INVALID_BRANCH"))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Couldn't find the snapshot for the branch INVALID_BRANCH");
+ }
+
+ @Test
+ public void testPartitionStatsOnUnPartitionedTable() throws Exception {
+ Table testTable =
+ TestTables.create(
+ tempDir("unpartitioned_table"),
+ "unpartitioned_table",
+ SCHEMA,
+ PartitionSpec.unpartitioned(),
+ 2);
+
+ List<Record> records = prepareRecords(testTable.schema());
+ DataFile dataFile = FileHelpers.writeDataFile(testTable, outputFile(), records);
+ testTable.newAppend().appendFile(dataFile).commit();
+
+ assertThatThrownBy(() -> PartitionStatsHandler.computeAndWriteStatsFile(testTable))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage("table must be partitioned");
+ }
+
+ @Test
+ public void testAllDatatypePartitionWriting() throws Exception {
+ Schema schema =
+ new Schema(
+ required(100, "id", Types.LongType.get()),
+ optional(101, "data", Types.StringType.get()),
+ required(102, "b", Types.BooleanType.get()),
+ optional(103, "i", Types.IntegerType.get()),
+ required(104, "l", Types.LongType.get()),
+ optional(105, "f", Types.FloatType.get()),
+ required(106, "d", Types.DoubleType.get()),
+ optional(107, "date", Types.DateType.get()),
+ required(108, "ts", Types.TimestampType.withoutZone()),
+ required(110, "s", Types.StringType.get()),
+ required(111, "uuid", Types.UUIDType.get()),
+ required(112, "fixed", Types.FixedType.ofLength(7)),
+ optional(113, "bytes", Types.BinaryType.get()),
+ required(114, "dec_9_0", Types.DecimalType.of(9, 0)),
+ required(115, "dec_11_2", Types.DecimalType.of(11, 2)),
+ required(116, "dec_38_10", Types.DecimalType.of(38, 10)), // maximum precision
+ required(117, "time", Types.TimeType.get()));
+
+ PartitionSpec spec =
+ PartitionSpec.builderFor(schema)
+ .identity("b")
+ .identity("i")
+ .identity("l")
+ .identity("f")
+ .identity("d")
+ .identity("date")
+ .identity("ts")
+ .identity("s")
+ .identity("uuid")
+ .identity("fixed")
+ .identity("bytes")
+ .identity("dec_9_0")
+ .identity("dec_11_2")
+ .identity("dec_38_10")
+ .identity("time")
+ .build();
+
+ Table testTable =
+ TestTables.create(
+ tempDir("test_all_type"), "test_all_type", schema, spec, SortOrder.unsorted(), 2);
+
+ Types.StructType partitionSchema = Partitioning.partitionType(testTable);
+ Schema dataSchema = PartitionStatsHandler.schema(partitionSchema);
+
+ Record partitionData =
+ GenericRecord.create(dataSchema.findField(Column.PARTITION.name()).type().asStructType());
+ partitionData.set(0, true);
+ partitionData.set(1, 42);
+ partitionData.set(2, 42L);
+ partitionData.set(3, 3.14f);
+ partitionData.set(4, 3.141592653589793);
+ partitionData.set(5, Literal.of("2022-01-01").to(Types.DateType.get()).value());
+ partitionData.set(
+ 6, Literal.of("2017-12-01T10:12:55.038194").to(Types.TimestampType.withoutZone()).value());
+ partitionData.set(7, "string");
+ partitionData.set(8, UUIDUtil.convertToByteBuffer(UUID.randomUUID()));
+ partitionData.set(9, new byte[] {0, 1, 2, 3, 4, 5, 6});
+ partitionData.set(10, new byte[] {1, 2, 3});
+ partitionData.set(11, Literal.of("123456789").to(Types.DecimalType.of(9, 0)).value());
+ partitionData.set(12, Literal.of("1234567.89").to(Types.DecimalType.of(11, 2)).value());
+ partitionData.set(
+ 13, Literal.of("12345678901234567890.1234567890").to(Types.DecimalType.of(38, 10)).value());
+ partitionData.set(14, Literal.of("10:10:10").to(Types.TimeType.get()).value());
+
+ PartitionStats partitionStats = new PartitionStats(partitionData, RANDOM.nextInt(10));
+ partitionStats.set(Column.DATA_RECORD_COUNT.id(), RANDOM.nextLong());
+ partitionStats.set(Column.DATA_FILE_COUNT.id(), RANDOM.nextInt());
+ partitionStats.set(Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(), 1024L * RANDOM.nextInt(20));
+
+ Iterator<PartitionStatsRecord> convertedRecords =
+ PartitionStatsHandler.statsToRecords(Collections.singletonList(partitionStats), dataSchema);
Review Comment:
There are already other tests below (for example `testPartitionStats`) that checks multiple rows/records.
This testcase is to check one row of "all possible data types" in partition values. I don't see the benefit of having more rows here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "regadas (via GitHub)" <gi...@apache.org>.
regadas commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2552425526
I just found this PR as I'm desperately looking for this functionality. Thanks @ajantha-bhat! Let's see if the review gets wrapped soon 🤞
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1851206461
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.BiFunction;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.data.avro.DataReader;
+import org.apache.iceberg.data.orc.GenericOrcReader;
+import org.apache.iceberg.data.parquet.GenericParquetReaders;
+import org.apache.iceberg.encryption.EncryptedFiles;
+import org.apache.iceberg.encryption.EncryptionKeyMetadata;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.FileWriterFactory;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.orc.ORC;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Schema schema = schema(partitionType);
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ Iterator<PartitionStatsRecord> convertedRecords = statsToRecords(sortedStats, schema);
+ return writePartitionStatsFile(table, currentSnapshot.snapshotId(), schema, convertedRecords);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStatsRecord> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+ FileWriterFactory<Record> factory =
+ GenericFileWriterFactory.builderFor(table)
+ .dataSchema(dataSchema)
+ .dataFileFormat(fileFormat(outputFile.location()))
+ .build();
+ DataWriter<Record> writer =
+ factory.newDataWriter(
+ EncryptedFiles.encryptedOutput(outputFile, EncryptionKeyMetadata.EMPTY),
+ PartitionSpec.unpartitioned(),
+ null);
+ try (Closeable toClose = writer) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStatsRecord> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<Record> records;
+ FileFormat fileFormat = fileFormat(inputFile.location());
+ switch (fileFormat) {
+ case PARQUET:
+ records =
+ Parquet.read(inputFile)
+ .project(schema)
+ .createReaderFunc(
+ fileSchema -> GenericParquetReaders.buildReader(schema, fileSchema))
+ .build();
+ break;
+ case ORC:
+ records =
+ ORC.read(inputFile)
+ .project(schema)
+ .createReaderFunc(fileSchema -> GenericOrcReader.buildReader(schema, fileSchema))
+ .build();
+ break;
+ case AVRO:
+ records = Avro.read(inputFile).project(schema).createReaderFunc(DataReader::create).build();
+ break;
+ default:
+ throw new UnsupportedOperationException("Unsupported file format:" + fileFormat.name());
+ }
+
+ return CloseableIterable.transform(
+ records, PartitionStatsHandler::recordToPartitionStatsRecord);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static PartitionStatsRecord recordToPartitionStatsRecord(Record record) {
+ PartitionStats stats =
+ new PartitionStats(
+ record.get(Column.PARTITION.id(), StructLike.class),
+ record.get(Column.SPEC_ID.id(), Integer.class));
+ stats.set(Column.DATA_RECORD_COUNT.id(), record.get(Column.DATA_RECORD_COUNT.id(), Long.class));
+ stats.set(Column.DATA_FILE_COUNT.id(), record.get(Column.DATA_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(),
+ record.get(Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(), Long.class));
+ stats.set(
+ Column.POSITION_DELETE_RECORD_COUNT.id(),
+ record.get(Column.POSITION_DELETE_RECORD_COUNT.id(), Long.class));
+ stats.set(
+ Column.POSITION_DELETE_FILE_COUNT.id(),
+ record.get(Column.POSITION_DELETE_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.EQUALITY_DELETE_RECORD_COUNT.id(),
+ record.get(Column.EQUALITY_DELETE_RECORD_COUNT.id(), Long.class));
+ stats.set(
+ Column.EQUALITY_DELETE_FILE_COUNT.id(),
+ record.get(Column.EQUALITY_DELETE_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.TOTAL_RECORD_COUNT.id(), record.get(Column.TOTAL_RECORD_COUNT.id(), Long.class));
+ stats.set(Column.LAST_UPDATED_AT.id(), record.get(Column.LAST_UPDATED_AT.id(), Long.class));
+ stats.set(
+ Column.LAST_UPDATED_SNAPSHOT_ID.id(),
+ record.get(Column.LAST_UPDATED_SNAPSHOT_ID.id(), Long.class));
+
+ return PartitionStatsRecord.create(record.struct(), stats);
+ }
+
+ @VisibleForTesting
+ static Iterator<PartitionStatsRecord> statsToRecords(
Review Comment:
> Let's at least estimate the amount of work to get the internal writer for Avro
This PR needs internal writer for parquet and orc as well not just Avro.
Considering 1.8.0 is planned end of this month and we have holidays coming up next month, I don't want to miss the release train again (like 1.7.0)
We are waiting for partition stats from long time (almost an year) and this PR is implemented based on what is available in the current Iceberg. I too agree that having internal writers will be nice. But it can be added in future and PR is designed such that we can replace current writers with internal writers without losing compatibility.
So, I don't see a reason to block the development of this feature.
Merging this PR will complete milestone for partition stats.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1907057882
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.BiFunction;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.data.avro.DataReader;
+import org.apache.iceberg.data.orc.GenericOrcReader;
+import org.apache.iceberg.data.parquet.GenericParquetReaders;
+import org.apache.iceberg.encryption.EncryptedFiles;
+import org.apache.iceberg.encryption.EncryptionKeyMetadata;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.FileWriterFactory;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.orc.ORC;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
Review Comment:
Wouldn't it be better not to have fat methods with multiple responsibilities? What if we introduce the write method that gets the stats iterator as an argument.
We might not need to execute the complete stats rebuild for all the registered partitions, but only for those that were changed in the current snapshot.
Snapshot summary already has a metric for the number of changed partitions, maybe we could extend it with the partition list and re-compute stats only for them. Then generate a new stats file based on the prev snapshot with updates to the changed. partitions.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1950048200
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records)
+ throws IOException {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile)) {
+ records.forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(Schema dataSchema, OutputFile outputFile)
+ throws IOException {
+ FileFormat fileFormat = fileFormat(outputFile.location());
Review Comment:
Didn't want to pass extra arguments. Wanted to keep signature similar to reader.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1960059571
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
Review Comment:
I have recently bumped into this:
https://github.com/apache/iceberg/blob/main/api/src/main/java/org/apache/iceberg/FileFormat.java#L60
```
public static FileFormat fromFileName(CharSequence filename) {
for (FileFormat format : VALUES) {
int extStart = filename.length() - format.ext.length();
if (Comparators.charSequences()
.compare(format.ext, filename.subSequence(extStart, filename.length()))
== 0) {
return format;
}
}
return null;
}
```
I still think this is brittle, but since it is used elsewhere in the code we can use the same method to get the file format from the filename.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1936923827
##########
data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java:
##########
@@ -0,0 +1,582 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.data.PartitionStatsHandler.Column;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionData;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assumptions;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestPartitionStatsHandler {
+ private static final Schema SCHEMA =
+ new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get()));
+
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("c2").identity("c3").build();
+
+ @TempDir public File temp;
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameters(name = "fileFormat = {0}")
+ public static List<Object> parameters() {
+ return Arrays.asList(FileFormat.PARQUET, FileFormat.ORC, FileFormat.AVRO);
+ }
+
+ @Parameter private FileFormat format;
+
+ @Test
+ public void testPartitionStatsOnEmptyTable() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_table"), "empty_table", SCHEMA, SPEC, 2);
+ assertThat(PartitionStatsHandler.computeAndWriteStatsFile(testTable)).isNull();
+ }
+
+ @Test
+ public void testPartitionStatsOnEmptyBranch() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_branch"), "empty_branch", SCHEMA, SPEC, 2);
+ testTable.manageSnapshots().createBranch("b1").commit();
+ PartitionStatisticsFile partitionStatisticsFile =
+ PartitionStatsHandler.computeAndWriteStatsFile(testTable, "b1");
+ // creates an empty stats file since the dummy snapshot exist
+ assertThat(partitionStatisticsFile.fileSizeInBytes()).isEqualTo(0L);
Review Comment:
test would be broken if default format changes, for example with avro format non-zero file would be created
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1939012670
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
Review Comment:
Any reason for having this `final`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1950039374
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records)
Review Comment:
Changed to `Iterable` instead of `Iterator`, because reader returns an `Iterable`. Just making similar signatures for reader and the writer.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "RussellSpitzer (via GitHub)" <gi...@apache.org>.
RussellSpitzer commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1817032978
##########
core/src/main/java/org/apache/iceberg/data/PartitionStatsRecord.java:
##########
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.relocated.com.google.common.base.Objects;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.types.Types.StructType;
+
+/** Wraps the {@link PartitionStats} as {@link Record}. Used by generic writers and readers. */
+public class PartitionStatsRecord implements Record, StructLike {
+ private static final LoadingCache<StructType, Map<String, Integer>> NAME_MAP_CACHE =
Review Comment:
I'm a little confused why we need a special class for this? GenericRecord should work right? Also Record already implements StructLike so that's unnecessary
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2623938276
Ping.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2614035260
@deniskuzZ: While designing the spec (https://iceberg.apache.org/spec/#partition-statistics-file), we have added `totalRecordCount` to represent the record count after applying the delete file. It is optional field and hence not computed at the moment as it requires scanning all the data files and it can be expensive operation.
> Could we extends the PartitionStats with totalFileSizeInBytes metric?
Let us wait for the merge of this PR. After that we can open the discussion to add additional stats for partition stats spec. For example some folks want min max stats also https://github.com/apache/iceberg/issues/11083.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1961367545
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
Review Comment:
Why is this resolved?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2668800302
@pvary, @gaborkaszab: I have addressed the comments. Please check.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1993387221
##########
core/src/main/java/org/apache/iceberg/SetPartitionStatistics.java:
##########
@@ -36,7 +35,10 @@ public SetPartitionStatistics(TableOperations ops) {
@Override
public UpdatePartitionStatistics setPartitionStatistics(PartitionStatisticsFile file) {
- Preconditions.checkArgument(null != file, "partition statistics file must not be null");
+ if (file == null) {
Review Comment:
We went little back and forth on this.
For the end users, it avoids extra null check if the stats are null by going with no-op. So, went ahead with no-op
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2690823724
Thanks everyone for the review and merging it.
This was a long awaited and super impactful feature. Hive, Trino, Dremio is waiting for it to be shipped.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1998023614
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
Review Comment:
I can't predefine the field for
`Types.NestedField partition = NestedField.required(1, Column.PARTITION.name(), unifiedPartitionType);`
because the `unifiedPartitionType` is a variable.
For other fields, I can define the `NestedField`, but for this I can't. So, it won't become uniform. I still feel Enum is cleaner. But I can modify if you guys really want it to be.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2632944341
thanks @pvary for the review.
I have addressed the comments except (https://github.com/apache/iceberg/pull/11216#discussion_r1939040414), It is because I wanted to keep the signatures of reader and writer similar. Let me know if you have any ideas. Thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1939030560
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
Review Comment:
nit: remove `;`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1939049410
##########
data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java:
##########
@@ -0,0 +1,582 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.data.PartitionStatsHandler.Column;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionData;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assumptions;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestPartitionStatsHandler {
+ private static final Schema SCHEMA =
+ new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get()));
+
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("c2").identity("c3").build();
+
+ @TempDir public File temp;
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameters(name = "fileFormat = {0}")
+ public static List<Object> parameters() {
+ return Arrays.asList(FileFormat.PARQUET, FileFormat.ORC, FileFormat.AVRO);
+ }
+
+ @Parameter private FileFormat format;
+
+ @Test
+ public void testPartitionStatsOnEmptyTable() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_table"), "empty_table", SCHEMA, SPEC, 2);
Review Comment:
Are these tables cleaned up after the test methods?
If not, they leave a state for the tests which is a bad practice
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1818884220
##########
core/src/test/java/org/apache/iceberg/TestTables.java:
##########
@@ -93,6 +93,26 @@ public static TestTable create(
return new TestTable(ops, name, reporter);
}
+ public static TestTable create(
+ File temp,
+ String name,
+ Schema schema,
+ PartitionSpec spec,
+ int formatVersion,
+ Map<String, String> properties) {
+ TestTableOperations ops = new TestTableOperations(name, temp);
+ if (ops.current() != null) {
+ throw new AlreadyExistsException("Table %s already exists at location: %s", name, temp);
Review Comment:
Just followed other create methods. Example line 85
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1818418685
##########
core/src/main/java/org/apache/iceberg/PartitionStats.java:
##########
@@ -249,4 +250,45 @@ public <T> void set(int pos, T value) {
throw new UnsupportedOperationException("Unknown position: " + pos);
}
}
+
+ @Override
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ } else if (!(other instanceof PartitionStats)) {
+ return false;
+ }
+
+ PartitionStats that = (PartitionStats) other;
+ return Objects.equals(partition, that.partition)
Review Comment:
We are storing `Record` as `Partition` from `coercedPartitionRecord`.
Since `GenericRecord` has `equals` implemented, calling `Objects.equals` is working here.
Hence, I didn't add comparator logic.
But agree that need to understand the implementation logic, just by looking at this class, it looks like we need comparator logic. I can update it if it is necessary.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1777915925
##########
core/src/test/java/org/apache/iceberg/TestPartitionStatsUtil.java:
##########
@@ -370,17 +371,17 @@ public void testPartitionStatsWithSchemaEvolution() throws Exception {
snapshot2.snapshotId()));
}
- private static PartitionData partitionData(Types.StructType partitionType, String c2, String c3) {
- PartitionData partitionData = new PartitionData(partitionType);
- partitionData.set(0, c2);
- partitionData.set(1, c3);
- return partitionData;
+ private static StructLike partitionData(Types.StructType partitionType, String c2, String c3) {
+ GenericRecord record = GenericRecord.create(partitionType);
Review Comment:
Stores the `Record` instead of `PartitionData` in the core logic. So, testcase is updated to expect `Record`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1935900633
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
Review Comment:
Aah, it became redundant after internal writer refactoring. Updated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1935880312
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
Review Comment:
redundant `@throws`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1941192206
##########
data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java:
##########
@@ -0,0 +1,582 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.data.PartitionStatsHandler.Column;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionData;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assumptions;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestPartitionStatsHandler {
+ private static final Schema SCHEMA =
+ new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get()));
+
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("c2").identity("c3").build();
+
+ @TempDir public File temp;
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameters(name = "fileFormat = {0}")
+ public static List<Object> parameters() {
+ return Arrays.asList(FileFormat.PARQUET, FileFormat.ORC, FileFormat.AVRO);
+ }
+
+ @Parameter private FileFormat format;
+
+ @Test
+ public void testPartitionStatsOnEmptyTable() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_table"), "empty_table", SCHEMA, SPEC, 2);
Review Comment:
Cool.
I thought that you use different table name in every test just to avoid collision.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1940508650
##########
data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java:
##########
@@ -0,0 +1,582 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.data.PartitionStatsHandler.Column;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionData;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assumptions;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestPartitionStatsHandler {
+ private static final Schema SCHEMA =
+ new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get()));
+
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("c2").identity("c3").build();
+
+ @TempDir public File temp;
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameters(name = "fileFormat = {0}")
+ public static List<Object> parameters() {
+ return Arrays.asList(FileFormat.PARQUET, FileFormat.ORC, FileFormat.AVRO);
+ }
+
+ @Parameter private FileFormat format;
+
+ @Test
+ public void testPartitionStatsOnEmptyTable() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_table"), "empty_table", SCHEMA, SPEC, 2);
+ assertThat(PartitionStatsHandler.computeAndWriteStatsFile(testTable)).isNull();
+ }
+
+ @Test
+ public void testPartitionStatsOnEmptyBranch() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_branch"), "empty_branch", SCHEMA, SPEC, 2);
+ testTable.manageSnapshots().createBranch("b1").commit();
+ PartitionStatisticsFile partitionStatisticsFile =
+ PartitionStatsHandler.computeAndWriteStatsFile(testTable, "b1");
+ // creates an empty stats file since the dummy snapshot exist
+ assertThat(partitionStatisticsFile.fileSizeInBytes()).isEqualTo(0L);
Review Comment:
Thinking more on it, behavior should be same as empty table testcase above. So, will update to not throw exceptions in this case.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1940531115
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
Review Comment:
This is private method and it is based on `newPartitionStatsFile`. It was a way to avoid passing more arguments.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "jbonofre (via GitHub)" <gi...@apache.org>.
jbonofre commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1852026216
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.BiFunction;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.data.avro.DataReader;
+import org.apache.iceberg.data.orc.GenericOrcReader;
+import org.apache.iceberg.data.parquet.GenericParquetReaders;
+import org.apache.iceberg.encryption.EncryptedFiles;
+import org.apache.iceberg.encryption.EncryptionKeyMetadata;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.FileWriterFactory;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.orc.ORC;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Schema schema = schema(partitionType);
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ Iterator<PartitionStatsRecord> convertedRecords = statsToRecords(sortedStats, schema);
+ return writePartitionStatsFile(table, currentSnapshot.snapshotId(), schema, convertedRecords);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStatsRecord> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+ FileWriterFactory<Record> factory =
+ GenericFileWriterFactory.builderFor(table)
+ .dataSchema(dataSchema)
+ .dataFileFormat(fileFormat(outputFile.location()))
+ .build();
+ DataWriter<Record> writer =
+ factory.newDataWriter(
+ EncryptedFiles.encryptedOutput(outputFile, EncryptionKeyMetadata.EMPTY),
+ PartitionSpec.unpartitioned(),
+ null);
+ try (Closeable toClose = writer) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStatsRecord> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<Record> records;
+ FileFormat fileFormat = fileFormat(inputFile.location());
+ switch (fileFormat) {
+ case PARQUET:
+ records =
+ Parquet.read(inputFile)
+ .project(schema)
+ .createReaderFunc(
+ fileSchema -> GenericParquetReaders.buildReader(schema, fileSchema))
+ .build();
+ break;
+ case ORC:
+ records =
+ ORC.read(inputFile)
+ .project(schema)
+ .createReaderFunc(fileSchema -> GenericOrcReader.buildReader(schema, fileSchema))
+ .build();
+ break;
+ case AVRO:
+ records = Avro.read(inputFile).project(schema).createReaderFunc(DataReader::create).build();
+ break;
+ default:
+ throw new UnsupportedOperationException("Unsupported file format:" + fileFormat.name());
+ }
+
+ return CloseableIterable.transform(
+ records, PartitionStatsHandler::recordToPartitionStatsRecord);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static PartitionStatsRecord recordToPartitionStatsRecord(Record record) {
+ PartitionStats stats =
+ new PartitionStats(
+ record.get(Column.PARTITION.id(), StructLike.class),
+ record.get(Column.SPEC_ID.id(), Integer.class));
+ stats.set(Column.DATA_RECORD_COUNT.id(), record.get(Column.DATA_RECORD_COUNT.id(), Long.class));
+ stats.set(Column.DATA_FILE_COUNT.id(), record.get(Column.DATA_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(),
+ record.get(Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(), Long.class));
+ stats.set(
+ Column.POSITION_DELETE_RECORD_COUNT.id(),
+ record.get(Column.POSITION_DELETE_RECORD_COUNT.id(), Long.class));
+ stats.set(
+ Column.POSITION_DELETE_FILE_COUNT.id(),
+ record.get(Column.POSITION_DELETE_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.EQUALITY_DELETE_RECORD_COUNT.id(),
+ record.get(Column.EQUALITY_DELETE_RECORD_COUNT.id(), Long.class));
+ stats.set(
+ Column.EQUALITY_DELETE_FILE_COUNT.id(),
+ record.get(Column.EQUALITY_DELETE_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.TOTAL_RECORD_COUNT.id(), record.get(Column.TOTAL_RECORD_COUNT.id(), Long.class));
+ stats.set(Column.LAST_UPDATED_AT.id(), record.get(Column.LAST_UPDATED_AT.id(), Long.class));
+ stats.set(
+ Column.LAST_UPDATED_SNAPSHOT_ID.id(),
+ record.get(Column.LAST_UPDATED_SNAPSHOT_ID.id(), Long.class));
+
+ return PartitionStatsRecord.create(record.struct(), stats);
+ }
+
+ @VisibleForTesting
+ static Iterator<PartitionStatsRecord> statsToRecords(
Review Comment:
Do we really need internal writers here ? I understand it's an improvement, but I don't think it's a blocker for this PR. The effort is ongoing for a long time now, I would be more in favor of moving forward soon and plan internal writers improvement in a second step.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1851202734
##########
core/src/main/java/org/apache/iceberg/PartitionStats.java:
##########
@@ -205,6 +211,8 @@ public <T> T get(int pos, Class<T> javaClass) {
public <T> void set(int pos, T value) {
switch (pos) {
case 0:
+ Preconditions.checkArgument(
Review Comment:
I can keep the member as `Record` instead of `StructLike` to avoid this. But since we have plan to use internal writers in future (which uses `StructLike`), we lose compatibility if we keep the member as `Record` instead of `StructLike`.
I don't think it is too awkward as `Record` implements `StructLike`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1998031053
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
Review Comment:
ok. We have similar non-uniform code in `DataFile` also for this kind.
https://github.com/apache/iceberg/blob/fcea78fc3571063fa172edd96be00b1fab0ba68e/api/src/main/java/org/apache/iceberg/DataFile.java#L123
I will raise a PR to have it similar way.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "gaborkaszab (via GitHub)" <gi...@apache.org>.
gaborkaszab commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1952317886
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records)
+ throws IOException {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile)) {
+ records.forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(Schema dataSchema, OutputFile outputFile)
+ throws IOException {
+ FileFormat fileFormat = fileFormat(outputFile.location());
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(InternalWriter::create)
+ .overwrite()
Review Comment:
> Snapshot id will be unique for each new commit. If users are trying to compute for stats for the same snapshot. It is overwriting the file as the contents doesn't change.
I'm still not sure about this. Let's assume we update the stats for a previous snapshot that already had stats. Also assume that there is a reader actually reading that stat file in the same time. In this case overwriting the file while the reader is reading it (e.g. reading the footer) might result in reading incorrectly, right? There might be an instant when the reader tries to read from the file but the file is not there. Or does Iceberg have any guarantees that this couldn't happen with an overwrite?
> the contents doesn't change
Also, it's possible that we don't write the very same content into the stat file. E.g. when stat files are corrupted for a reason and we want to re-compute them.
Another use-case that I have in mind for writing different content into an existing file is when we decide to extend the stats schema with some additional columns and then we decide to re-compute the stats to have the new column types too in the file. Not saying that this is a scenario that happens frequently but it's also not impossible.
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
Review Comment:
Just thinking out loud. With returning null we won't be able to have this easy usage of the functionality:
`testTable.updatePartitionStatistics().setPartitionStatistics(PartitionStatsHandler.computeAndWriteStatsFile(testTable, "b1");).commit();`
Not sure how common it is in Iceberg to do null checks after trying to create some resources. For instance I recall table creation doesn't give null to users but a Table object or it throws if it didn't succeed creating one. Throwing an exception that we weren't able to create stats might make sense. Still not confident with either approach.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1950032898
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
Review Comment:
from the spec: https://iceberg.apache.org/spec/#partition-statistics-file
> Accurate count of records in a partition after applying the delete files if any
We also recently discussed this in https://github.com/apache/iceberg/pull/12098#discussion_r1940551771
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2690641023
Thanks for all your work @ajantha-bhat!
Also thanks for all of the reviewers!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1961376054
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records)
+ throws IOException {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile)) {
+ records.forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(Schema dataSchema, OutputFile outputFile)
+ throws IOException {
+ FileFormat fileFormat = fileFormat(outputFile.location());
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(InternalWriter::create)
+ .overwrite()
Review Comment:
I agree with @gaborkaszab that we need to handle when the stats file is recreated. Otherwise we might create a situation where we remove a stat file during some reader tries to access it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1818884220
##########
core/src/test/java/org/apache/iceberg/TestTables.java:
##########
@@ -93,6 +93,26 @@ public static TestTable create(
return new TestTable(ops, name, reporter);
}
+ public static TestTable create(
+ File temp,
+ String name,
+ Schema schema,
+ PartitionSpec spec,
+ int formatVersion,
+ Map<String, String> properties) {
+ TestTableOperations ops = new TestTableOperations(name, temp);
+ if (ops.current() != null) {
+ throw new AlreadyExistsException("Table %s already exists at location: %s", name, temp);
Review Comment:
Just followed other create methods in this file. Example line 85
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1961038669
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
Review Comment:
Thanks. I have updated the code to use `FileFormat.fromFileName`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2739877960
@deniskuzZ : Hi, Is Hive intersted in the incremental compute of these partition stats instead of reading all the manifests at once? If so, I am planning to add a new API to incrementally compute stats (it checks the last snapshot that had partition stats and compute the difference manifest from that snapshot to current snapshot and compute new stas by reading the old stats + new stats from new manifests)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1857583188
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.BiFunction;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.data.avro.DataReader;
+import org.apache.iceberg.data.orc.GenericOrcReader;
+import org.apache.iceberg.data.parquet.GenericParquetReaders;
+import org.apache.iceberg.encryption.EncryptedFiles;
+import org.apache.iceberg.encryption.EncryptionKeyMetadata;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.FileWriterFactory;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.orc.ORC;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Schema schema = schema(partitionType);
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ Iterator<PartitionStatsRecord> convertedRecords = statsToRecords(sortedStats, schema);
+ return writePartitionStatsFile(table, currentSnapshot.snapshotId(), schema, convertedRecords);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStatsRecord> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+ FileWriterFactory<Record> factory =
+ GenericFileWriterFactory.builderFor(table)
+ .dataSchema(dataSchema)
+ .dataFileFormat(fileFormat(outputFile.location()))
+ .build();
+ DataWriter<Record> writer =
+ factory.newDataWriter(
+ EncryptedFiles.encryptedOutput(outputFile, EncryptionKeyMetadata.EMPTY),
+ PartitionSpec.unpartitioned(),
+ null);
+ try (Closeable toClose = writer) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStatsRecord> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<Record> records;
+ FileFormat fileFormat = fileFormat(inputFile.location());
+ switch (fileFormat) {
+ case PARQUET:
+ records =
+ Parquet.read(inputFile)
+ .project(schema)
+ .createReaderFunc(
+ fileSchema -> GenericParquetReaders.buildReader(schema, fileSchema))
+ .build();
+ break;
+ case ORC:
+ records =
+ ORC.read(inputFile)
+ .project(schema)
+ .createReaderFunc(fileSchema -> GenericOrcReader.buildReader(schema, fileSchema))
+ .build();
+ break;
+ case AVRO:
+ records = Avro.read(inputFile).project(schema).createReaderFunc(DataReader::create).build();
+ break;
+ default:
+ throw new UnsupportedOperationException("Unsupported file format:" + fileFormat.name());
+ }
+
+ return CloseableIterable.transform(
+ records, PartitionStatsHandler::recordToPartitionStatsRecord);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static PartitionStatsRecord recordToPartitionStatsRecord(Record record) {
+ PartitionStats stats =
+ new PartitionStats(
+ record.get(Column.PARTITION.id(), StructLike.class),
+ record.get(Column.SPEC_ID.id(), Integer.class));
+ stats.set(Column.DATA_RECORD_COUNT.id(), record.get(Column.DATA_RECORD_COUNT.id(), Long.class));
+ stats.set(Column.DATA_FILE_COUNT.id(), record.get(Column.DATA_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(),
+ record.get(Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(), Long.class));
+ stats.set(
+ Column.POSITION_DELETE_RECORD_COUNT.id(),
+ record.get(Column.POSITION_DELETE_RECORD_COUNT.id(), Long.class));
+ stats.set(
+ Column.POSITION_DELETE_FILE_COUNT.id(),
+ record.get(Column.POSITION_DELETE_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.EQUALITY_DELETE_RECORD_COUNT.id(),
+ record.get(Column.EQUALITY_DELETE_RECORD_COUNT.id(), Long.class));
+ stats.set(
+ Column.EQUALITY_DELETE_FILE_COUNT.id(),
+ record.get(Column.EQUALITY_DELETE_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.TOTAL_RECORD_COUNT.id(), record.get(Column.TOTAL_RECORD_COUNT.id(), Long.class));
+ stats.set(Column.LAST_UPDATED_AT.id(), record.get(Column.LAST_UPDATED_AT.id(), Long.class));
+ stats.set(
+ Column.LAST_UPDATED_SNAPSHOT_ID.id(),
+ record.get(Column.LAST_UPDATED_SNAPSHOT_ID.id(), Long.class));
+
+ return PartitionStatsRecord.create(record.struct(), stats);
+ }
+
+ @VisibleForTesting
+ static Iterator<PartitionStatsRecord> statsToRecords(
Review Comment:
ping @rdblue, @aokolnychyi, @RussellSpitzer
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1916789379
##########
core/src/main/java/org/apache/iceberg/PartitionStatsUtil.java:
##########
@@ -133,4 +134,27 @@ private static Collection<PartitionStats> mergeStats(
return statsMap.values();
}
+
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ public static boolean isEqual(
+ Comparator<StructLike> partitionComparator, PartitionStats stats1, PartitionStats stats2) {
Review Comment:
Cannot have `Equals` and `HashCode` for `PartitionStats` class as `StructLike` need to have comparator for `equals()` which forces that class extends `StructLike` to hold some more things. Setting comparator while serializing and deserializing that class will be a mess.
Hence, added this util method. Currently used only by tests. But can be useful for developers when they integrate partition stats to engines, they can use it for their tests. So, kept as a util.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1941247104
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
Review Comment:
It would be better to add the fileformat to the `PartitionStatisticsFile`.
Inferring FileFormat based on the extension is a bad practice
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1939057416
##########
data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java:
##########
@@ -0,0 +1,582 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.data.PartitionStatsHandler.Column;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionData;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assumptions;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestPartitionStatsHandler {
+ private static final Schema SCHEMA =
+ new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get()));
+
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("c2").identity("c3").build();
+
+ @TempDir public File temp;
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameters(name = "fileFormat = {0}")
+ public static List<Object> parameters() {
+ return Arrays.asList(FileFormat.PARQUET, FileFormat.ORC, FileFormat.AVRO);
+ }
+
+ @Parameter private FileFormat format;
+
+ @Test
+ public void testPartitionStatsOnEmptyTable() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_table"), "empty_table", SCHEMA, SPEC, 2);
+ assertThat(PartitionStatsHandler.computeAndWriteStatsFile(testTable)).isNull();
+ }
+
+ @Test
+ public void testPartitionStatsOnEmptyBranch() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_branch"), "empty_branch", SCHEMA, SPEC, 2);
+ testTable.manageSnapshots().createBranch("b1").commit();
+ PartitionStatisticsFile partitionStatisticsFile =
+ PartitionStatsHandler.computeAndWriteStatsFile(testTable, "b1");
+ // creates an empty stats file since the dummy snapshot exist
+ assertThat(partitionStatisticsFile.fileSizeInBytes()).isEqualTo(0L);
Review Comment:
I think it is bad practice to check in tests for things which are not a requirement, just "coincidentally" happens.
Could we check that the file could be read and actually emtpy?
##########
data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java:
##########
@@ -0,0 +1,582 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.data.PartitionStatsHandler.Column;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionData;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assumptions;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestPartitionStatsHandler {
+ private static final Schema SCHEMA =
+ new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get()));
+
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("c2").identity("c3").build();
+
+ @TempDir public File temp;
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameters(name = "fileFormat = {0}")
+ public static List<Object> parameters() {
+ return Arrays.asList(FileFormat.PARQUET, FileFormat.ORC, FileFormat.AVRO);
+ }
+
+ @Parameter private FileFormat format;
+
+ @Test
+ public void testPartitionStatsOnEmptyTable() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_table"), "empty_table", SCHEMA, SPEC, 2);
+ assertThat(PartitionStatsHandler.computeAndWriteStatsFile(testTable)).isNull();
+ }
+
+ @Test
+ public void testPartitionStatsOnEmptyBranch() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_branch"), "empty_branch", SCHEMA, SPEC, 2);
+ testTable.manageSnapshots().createBranch("b1").commit();
+ PartitionStatisticsFile partitionStatisticsFile =
+ PartitionStatsHandler.computeAndWriteStatsFile(testTable, "b1");
+ // creates an empty stats file since the dummy snapshot exist
+ assertThat(partitionStatisticsFile.fileSizeInBytes()).isEqualTo(0L);
Review Comment:
I think it is bad practice to check in tests for things which are not a requirement, just "coincidentally" happens.
Could we check that the file could be read and actually empty?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1940542661
##########
data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java:
##########
@@ -0,0 +1,582 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.data.PartitionStatsHandler.Column;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionData;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assumptions;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestPartitionStatsHandler {
+ private static final Schema SCHEMA =
+ new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get()));
+
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("c2").identity("c3").build();
+
+ @TempDir public File temp;
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameters(name = "fileFormat = {0}")
+ public static List<Object> parameters() {
+ return Arrays.asList(FileFormat.PARQUET, FileFormat.ORC, FileFormat.AVRO);
+ }
+
+ @Parameter private FileFormat format;
+
+ @Test
+ public void testPartitionStatsOnEmptyTable() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_table"), "empty_table", SCHEMA, SPEC, 2);
Review Comment:
It is similar to other existing tests. @TempDir annotation should clean up the folders.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1995413058
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
Review Comment:
I think @aokolnychyi suggests to use constants for the fields instead of the field names.
Checked the codebase, and this is the typical pattern.
See:
- https://github.com/apache/iceberg/blob/main/core/src/main/java/org/apache/iceberg/MetadataColumns.java
- https://github.com/apache/iceberg/blob/main/core/src/main/java/org/apache/iceberg/ManifestEntry.java
- https://github.com/apache/iceberg/blob/main/api/src/main/java/org/apache/iceberg/DataFile.java
I agree that we should follow the same pattern.
@ajantha-bhat: Could you please create a PR for it?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1950023950
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
Review Comment:
Keeping short names to avoid long lines in usage of the enum.
Column.PARTITION.id() is still meaningful.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1950014481
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
Review Comment:
> It would be better to add the fileformat to the PartitionStatisticsFile.
I like it. Similar to the field in `data_file` spec.
This would be a spec modification in https://iceberg.apache.org/spec/#partition-statistics
@aokolnychyi: WDYT?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1851216659
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.BiFunction;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.data.avro.DataReader;
+import org.apache.iceberg.data.orc.GenericOrcReader;
+import org.apache.iceberg.data.parquet.GenericParquetReaders;
+import org.apache.iceberg.encryption.EncryptedFiles;
+import org.apache.iceberg.encryption.EncryptionKeyMetadata;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.FileWriterFactory;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.orc.ORC;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Schema schema = schema(partitionType);
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ Iterator<PartitionStatsRecord> convertedRecords = statsToRecords(sortedStats, schema);
+ return writePartitionStatsFile(table, currentSnapshot.snapshotId(), schema, convertedRecords);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStatsRecord> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+ FileWriterFactory<Record> factory =
+ GenericFileWriterFactory.builderFor(table)
+ .dataSchema(dataSchema)
+ .dataFileFormat(fileFormat(outputFile.location()))
+ .build();
+ DataWriter<Record> writer =
+ factory.newDataWriter(
+ EncryptedFiles.encryptedOutput(outputFile, EncryptionKeyMetadata.EMPTY),
+ PartitionSpec.unpartitioned(),
+ null);
+ try (Closeable toClose = writer) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStatsRecord> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<Record> records;
+ FileFormat fileFormat = fileFormat(inputFile.location());
+ switch (fileFormat) {
+ case PARQUET:
+ records =
+ Parquet.read(inputFile)
+ .project(schema)
+ .createReaderFunc(
+ fileSchema -> GenericParquetReaders.buildReader(schema, fileSchema))
+ .build();
+ break;
+ case ORC:
+ records =
+ ORC.read(inputFile)
+ .project(schema)
+ .createReaderFunc(fileSchema -> GenericOrcReader.buildReader(schema, fileSchema))
+ .build();
+ break;
+ case AVRO:
+ records = Avro.read(inputFile).project(schema).createReaderFunc(DataReader::create).build();
+ break;
+ default:
+ throw new UnsupportedOperationException("Unsupported file format:" + fileFormat.name());
+ }
+
+ return CloseableIterable.transform(
+ records, PartitionStatsHandler::recordToPartitionStatsRecord);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static PartitionStatsRecord recordToPartitionStatsRecord(Record record) {
+ PartitionStats stats =
+ new PartitionStats(
+ record.get(Column.PARTITION.id(), StructLike.class),
+ record.get(Column.SPEC_ID.id(), Integer.class));
+ stats.set(Column.DATA_RECORD_COUNT.id(), record.get(Column.DATA_RECORD_COUNT.id(), Long.class));
+ stats.set(Column.DATA_FILE_COUNT.id(), record.get(Column.DATA_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(),
+ record.get(Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(), Long.class));
+ stats.set(
+ Column.POSITION_DELETE_RECORD_COUNT.id(),
+ record.get(Column.POSITION_DELETE_RECORD_COUNT.id(), Long.class));
+ stats.set(
+ Column.POSITION_DELETE_FILE_COUNT.id(),
+ record.get(Column.POSITION_DELETE_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.EQUALITY_DELETE_RECORD_COUNT.id(),
+ record.get(Column.EQUALITY_DELETE_RECORD_COUNT.id(), Long.class));
+ stats.set(
+ Column.EQUALITY_DELETE_FILE_COUNT.id(),
+ record.get(Column.EQUALITY_DELETE_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.TOTAL_RECORD_COUNT.id(), record.get(Column.TOTAL_RECORD_COUNT.id(), Long.class));
+ stats.set(Column.LAST_UPDATED_AT.id(), record.get(Column.LAST_UPDATED_AT.id(), Long.class));
+ stats.set(
+ Column.LAST_UPDATED_SNAPSHOT_ID.id(),
+ record.get(Column.LAST_UPDATED_SNAPSHOT_ID.id(), Long.class));
+
+ return PartitionStatsRecord.create(record.struct(), stats);
+ }
+
+ @VisibleForTesting
+ static Iterator<PartitionStatsRecord> statsToRecords(
Review Comment:
And regarding effort for internal writers, I tried a POC last time just for parquet.
https://github.com/ajantha-bhat/iceberg/commit/c209bc94c3a43e3fb8262cffa121dc450889dfd8
Introduced `GenericStructParquetWriter` used by `GenericStructFileWriterFactory`.
As It uses `BaseParquetWriter` which expects `LocalDate` for date type instead of `Int` type and so on for other types. Should we use `BaseParquetWriter` with converter for internal writers or we should go and refactor the `ParquetValueWriters` and `ColumnWriter` was my doubt.
Also, If we use writers with converters, `StructLike` comparator will fail as it need `int` type for date but the final value is `LocalDate`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1953427985
##########
core/src/test/java/org/apache/iceberg/TestTables.java:
##########
@@ -93,6 +93,26 @@ public static TestTable create(
return new TestTable(ops, name, reporter);
}
+ public static TestTable create(
+ File temp,
+ String name,
+ Schema schema,
+ PartitionSpec spec,
+ int formatVersion,
+ Map<String, String> properties) {
Review Comment:
Builder method is definitely better. I just extracted common code into a private function as of now to keep minimal changes in the PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2690498532
Just rebased the PR as there was a conflict from https://github.com/apache/iceberg/pull/12419 in `TestTables`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "jbonofre (via GitHub)" <gi...@apache.org>.
jbonofre commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2505384190
@RussellSpitzer @aokolnychyi I'm reviewing the stale PRs, and this one is open for month. Do we have a way to move forward ? I can do a new review, but at the end of the day, it won't help for the merge (as only committers can merge PR).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1940544107
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
Review Comment:
To avoid computing partition type again again in `computeAndWriteStatsFile`. Also, It is recommended to pass only what is required for the method instead of the whole table.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1937122625
##########
data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java:
##########
@@ -0,0 +1,582 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.data.PartitionStatsHandler.Column;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionData;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assumptions;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestPartitionStatsHandler {
+ private static final Schema SCHEMA =
+ new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get()));
+
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("c2").identity("c3").build();
+
+ @TempDir public File temp;
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameters(name = "fileFormat = {0}")
+ public static List<Object> parameters() {
+ return Arrays.asList(FileFormat.PARQUET, FileFormat.ORC, FileFormat.AVRO);
+ }
+
+ @Parameter private FileFormat format;
+
+ @Test
+ public void testPartitionStatsOnEmptyTable() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_table"), "empty_table", SCHEMA, SPEC, 2);
+ assertThat(PartitionStatsHandler.computeAndWriteStatsFile(testTable)).isNull();
+ }
+
+ @Test
+ public void testPartitionStatsOnEmptyBranch() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_branch"), "empty_branch", SCHEMA, SPEC, 2);
+ testTable.manageSnapshots().createBranch("b1").commit();
+ PartitionStatisticsFile partitionStatisticsFile =
+ PartitionStatsHandler.computeAndWriteStatsFile(testTable, "b1");
+ // creates an empty stats file since the dummy snapshot exist
+ assertThat(partitionStatisticsFile.fileSizeInBytes()).isEqualTo(0L);
Review Comment:
true, I've already added +1
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1939041619
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
Review Comment:
Do we need to check that the table implements `HasTableOperations`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1939040414
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
Review Comment:
The `fileFormat` method parameter is a `fileLocation`, here we provide the actual `FileFormat` string... this seems like an issue for me
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1939036304
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
Review Comment:
Are we sure in this?
We usually depend on metadata files to deduce the file format. Depending on the filename seems brittle to me.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1939011161
##########
core/src/test/java/org/apache/iceberg/TestTables.java:
##########
@@ -93,6 +93,26 @@ public static TestTable create(
return new TestTable(ops, name, reporter);
}
+ public static TestTable create(
+ File temp,
+ String name,
+ Schema schema,
+ PartitionSpec spec,
+ int formatVersion,
+ Map<String, String> properties) {
Review Comment:
Why not adding the parameter to the old `create` method and call the new method from the old one?
Like:
```
public static TestTable create(
File temp,
String name,
Schema schema,
PartitionSpec spec,
SortOrder sortOrder,
int formatVersion) {
return create(temp, name, schema, spec, SortOrder.unsorted(), formatVersion, ImmutableMap.of());
}
public static TestTable create(
File temp,
String name,
Schema schema,
PartitionSpec spec,
int formatVersion,
Map<String, String> properties) {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1940512306
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
Review Comment:
Should this be an exception? When we query empty table, it returns zero rows. Similarly, it returns null. I will update the java doc.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "gaborkaszab (via GitHub)" <gi...@apache.org>.
gaborkaszab commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1946313092
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
Review Comment:
nit: isn't this rather a ColumnType or similar?
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
Review Comment:
Just for my understanding: I'm pretty sure this has been discussed, but is this a total of the data + delete records? Isn't it redundant with those fields?
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
Review Comment:
nit: Maybe SnapshotRef.MAIN_BRANCH instead of null for more readability?
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records)
+ throws IOException {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile)) {
+ records.forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+ FileFormat fileFormat =
+ fileFormat(
Review Comment:
wouldn't FileFormat.fromString(...getOrDefault(...)) work here? No need to call the local function that accepts locations.
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records)
+ throws IOException {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile)) {
+ records.forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(Schema dataSchema, OutputFile outputFile)
+ throws IOException {
+ FileFormat fileFormat = fileFormat(outputFile.location());
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(InternalWriter::create)
+ .overwrite()
Review Comment:
Assume we already have a stats file and we overwrite it here. Wouldn't it cause any issues if someone is currently reading the file we overwrite now?
I see the Puffin stats get a UUID in their name (if written by the [Spark action](https://github.com/apache/iceberg/blob/main/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/ComputeTableStatsSparkAction.java#L176)). Well that introduces another set of problems since once updating an existing file and not overwriting it, we leave orphaned files behind.
WDYT?
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records)
+ throws IOException {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile)) {
+ records.forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(Schema dataSchema, OutputFile outputFile)
+ throws IOException {
+ FileFormat fileFormat = fileFormat(outputFile.location());
Review Comment:
for writing I think we can get the format from the TblProperties.
##########
core/src/test/java/org/apache/iceberg/TestTables.java:
##########
@@ -93,6 +93,26 @@ public static TestTable create(
return new TestTable(ops, name, reporter);
}
+ public static TestTable create(
+ File temp,
+ String name,
+ Schema schema,
+ PartitionSpec spec,
+ int formatVersion,
+ Map<String, String> properties) {
Review Comment:
FYI, at some point I was considering introducing a Builder class for these TestTables because there are som many different create() functions now. Once this PR is merged, I think I'll re-visit this plan since we have yet another version of these functions.
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
Review Comment:
I personally wouldn't bother with empty stats files. They just have no additional value and could make noise if we write them to storage.
However, how UpdatePartitionStatistics is implemented now, the user gonna have to make an extra null check or would get a Precondition failure if the stats file is null. So you won't be able to do this:
`tbl.updatePartitionStatistics(PartitionStatsHandler.computeAndWriteStatsFile(...))`
So we either have to have an extra null check on the user side or we'd get an exception anyway (if we don't want to write empty stats files). So why not throwing here then?
WDYT?
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
Review Comment:
Probably there is a reason, so mostly for my understanding: Isn't this schema is mostly a property of a PartitionStatisticsFile and not for a PartitionStatsHandler? I think logically the schema belongs to the file and not to the writer.
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records)
+ throws IOException {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile)) {
+ records.forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
Review Comment:
I find this misleading as we call this function not just for file locations but also for the file format table property.
##########
core/src/main/java/org/apache/iceberg/PartitionStats.java:
##########
@@ -18,12 +18,16 @@
*/
package org.apache.iceberg;
+import java.util.Objects;
+import org.apache.iceberg.data.Record;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
public class PartitionStats implements StructLike {
private static final int STATS_COUNT = 12;
+ // Storing as StructLike instead of Record as
Review Comment:
nit: line break seems odd here
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
Review Comment:
nit: first usage for `partitionType` is in L149. I'd move this closer.
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
Review Comment:
Same question as above I guess: null or exception or empty stats
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records)
Review Comment:
Is there a use case when we give an iterator to this function that doesn't point to the beginning of the record list? If not, I think I'd rather have a List parameter here and not the iterator to avoid misuse of this function.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2671333629
@pvary: Thanks a lot for the review and approval.
Yeah, we can wait till next week for @gaborkaszab. By that time @aokolnychyi may also review this if he gets time.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1818885035
##########
core/src/main/java/org/apache/iceberg/PartitionStats.java:
##########
@@ -249,4 +250,45 @@ public <T> void set(int pos, T value) {
throw new UnsupportedOperationException("Unknown position: " + pos);
}
}
+
+ @Override
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ } else if (!(other instanceof PartitionStats)) {
+ return false;
+ }
+
+ PartitionStats that = (PartitionStats) other;
+ return Objects.equals(partition, that.partition)
+ && specId == that.specId
+ && dataRecordCount == that.dataRecordCount
+ && dataFileCount == that.dataFileCount
+ && totalDataFileSizeInBytes == that.totalDataFileSizeInBytes
+ && positionDeleteRecordCount == that.positionDeleteRecordCount
+ && positionDeleteFileCount == that.positionDeleteFileCount
+ && equalityDeleteRecordCount == that.equalityDeleteRecordCount
+ && equalityDeleteFileCount == that.equalityDeleteFileCount
+ && totalRecordCount == that.totalRecordCount
+ && Objects.equals(lastUpdatedAt, that.lastUpdatedAt)
+ && Objects.equals(lastUpdatedSnapshotId, that.lastUpdatedSnapshotId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ partition,
Review Comment:
Replied above.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1777916406
##########
core/src/test/java/org/apache/iceberg/TestTables.java:
##########
@@ -93,6 +93,26 @@ public static TestTable create(
return new TestTable(ops, name, reporter);
}
+ public static TestTable create(
+ File temp,
+ String name,
+ Schema schema,
+ PartitionSpec spec,
+ int formatVersion,
+ Map<String, String> properties) {
Review Comment:
There was no option to pass the table properties before.
Needed to pass different file format for paramterized test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2613938136
hi @ajantha-bhat, what is the purpose of `PartitionStats.totalRecordCount`? it's always 0 and there is no external setter either.
Also `SnapshotSummary.TOTAL_FILE_SIZE_PROP` tracks all files (data + delete), whereas PartitionStats only data files `totalDataFileSizeInBytes`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "aokolnychyi (via GitHub)" <gi...@apache.org>.
aokolnychyi commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1992478432
##########
core/src/main/java/org/apache/iceberg/SetPartitionStatistics.java:
##########
@@ -36,7 +35,10 @@ public SetPartitionStatistics(TableOperations ops) {
@Override
public UpdatePartitionStatistics setPartitionStatistics(PartitionStatisticsFile file) {
- Preconditions.checkArgument(null != file, "partition statistics file must not be null");
+ if (file == null) {
Review Comment:
I actually think it was reasonable to throw an NPE and be consistent with `UpdateStatistics`. There is no problem with NPE as long as the error message is descriptive.
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a combined partition type which considers
+ * all specs in a table.
+ *
+ * @param unifiedPartitionType unified partition schema type. Could be calculated by {@link
+ * Partitioning#partitionType(Table)}.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType unifiedPartitionType) {
+ Preconditions.checkState(!unifiedPartitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), unifiedPartitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, SnapshotRef.MAIN_BRANCH);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null || branch.equals(SnapshotRef.MAIN_BRANCH),
+ "Couldn't find the snapshot for the branch %s",
+ branch);
+ return null;
+ }
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterable<PartitionStats> records)
+ throws IOException {
+ FileFormat fileFormat =
+ FileFormat.fromString(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+
+ OutputFile outputFile = newPartitionStatsFile(table, fileFormat, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile, fileFormat)) {
+ records.iterator().forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static OutputFile newPartitionStatsFile(
+ Table table, FileFormat fileFormat, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(
+ Locale.ROOT, "partition-stats-%d-%s", snapshotId, UUID.randomUUID()))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(
+ Schema dataSchema, OutputFile outputFile, FileFormat fileFormat) throws IOException {
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(InternalWriter::create)
+ .withSpec(PartitionSpec.unpartitioned())
+ .build();
+ case AVRO:
+ return Avro.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(org.apache.iceberg.avro.InternalWriter::create)
+ .withSpec(PartitionSpec.unpartitioned())
+ .build();
+ case ORC:
+ // Internal writers are not supported for ORC yet.
+ default:
+ throw new UnsupportedOperationException("Unsupported file format:" + fileFormat.name());
+ }
+ }
+
+ private static CloseableIterable<StructLike> dataReader(Schema schema, InputFile inputFile) {
+ FileFormat fileFormat = FileFormat.fromFileName(inputFile.location());
+ Preconditions.checkArgument(
+ fileFormat != null, "Unable to determine format of file: %s", inputFile.location());
+
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.read(inputFile)
+ .project(schema)
+ .createReaderFunc(
+ fileSchema ->
+ org.apache.iceberg.data.parquet.InternalReader.create(schema, fileSchema))
+ .build();
+ case AVRO:
+ return Avro.read(inputFile)
+ .project(schema)
+ .createReaderFunc(fileSchema -> InternalReader.create(schema))
+ .build();
+ case ORC:
+ // Internal readers are not supported for ORC yet.
+ default:
+ throw new UnsupportedOperationException("Unsupported file format:" + fileFormat.name());
+ }
+ }
+
+ private static PartitionStats recordToPartitionStats(StructLike record) {
+ PartitionStats stats =
Review Comment:
@ajantha-bhat, could you confirm whether `CloseableIterable<StructLike>` returned by the data reader reuses the same object? If so, we don't have to create a new `PartitionStats` object for every record.
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
Review Comment:
I wonder whether this enum is actually required. Why not simply define `NestedField` constants? We need those to create a schema anyway?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "aokolnychyi (via GitHub)" <gi...@apache.org>.
aokolnychyi commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1851155557
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.BiFunction;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.data.avro.DataReader;
+import org.apache.iceberg.data.orc.GenericOrcReader;
+import org.apache.iceberg.data.parquet.GenericParquetReaders;
+import org.apache.iceberg.encryption.EncryptedFiles;
+import org.apache.iceberg.encryption.EncryptionKeyMetadata;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.FileWriterFactory;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.orc.ORC;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Schema schema = schema(partitionType);
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ Iterator<PartitionStatsRecord> convertedRecords = statsToRecords(sortedStats, schema);
+ return writePartitionStatsFile(table, currentSnapshot.snapshotId(), schema, convertedRecords);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStatsRecord> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+ FileWriterFactory<Record> factory =
+ GenericFileWriterFactory.builderFor(table)
+ .dataSchema(dataSchema)
+ .dataFileFormat(fileFormat(outputFile.location()))
+ .build();
+ DataWriter<Record> writer =
+ factory.newDataWriter(
+ EncryptedFiles.encryptedOutput(outputFile, EncryptionKeyMetadata.EMPTY),
+ PartitionSpec.unpartitioned(),
+ null);
+ try (Closeable toClose = writer) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStatsRecord> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<Record> records;
+ FileFormat fileFormat = fileFormat(inputFile.location());
+ switch (fileFormat) {
+ case PARQUET:
+ records =
+ Parquet.read(inputFile)
+ .project(schema)
+ .createReaderFunc(
+ fileSchema -> GenericParquetReaders.buildReader(schema, fileSchema))
+ .build();
+ break;
+ case ORC:
+ records =
+ ORC.read(inputFile)
+ .project(schema)
+ .createReaderFunc(fileSchema -> GenericOrcReader.buildReader(schema, fileSchema))
+ .build();
+ break;
+ case AVRO:
+ records = Avro.read(inputFile).project(schema).createReaderFunc(DataReader::create).build();
+ break;
+ default:
+ throw new UnsupportedOperationException("Unsupported file format:" + fileFormat.name());
+ }
+
+ return CloseableIterable.transform(
+ records, PartitionStatsHandler::recordToPartitionStatsRecord);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static PartitionStatsRecord recordToPartitionStatsRecord(Record record) {
+ PartitionStats stats =
+ new PartitionStats(
+ record.get(Column.PARTITION.id(), StructLike.class),
+ record.get(Column.SPEC_ID.id(), Integer.class));
+ stats.set(Column.DATA_RECORD_COUNT.id(), record.get(Column.DATA_RECORD_COUNT.id(), Long.class));
+ stats.set(Column.DATA_FILE_COUNT.id(), record.get(Column.DATA_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(),
+ record.get(Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(), Long.class));
+ stats.set(
+ Column.POSITION_DELETE_RECORD_COUNT.id(),
+ record.get(Column.POSITION_DELETE_RECORD_COUNT.id(), Long.class));
+ stats.set(
+ Column.POSITION_DELETE_FILE_COUNT.id(),
+ record.get(Column.POSITION_DELETE_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.EQUALITY_DELETE_RECORD_COUNT.id(),
+ record.get(Column.EQUALITY_DELETE_RECORD_COUNT.id(), Long.class));
+ stats.set(
+ Column.EQUALITY_DELETE_FILE_COUNT.id(),
+ record.get(Column.EQUALITY_DELETE_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.TOTAL_RECORD_COUNT.id(), record.get(Column.TOTAL_RECORD_COUNT.id(), Long.class));
+ stats.set(Column.LAST_UPDATED_AT.id(), record.get(Column.LAST_UPDATED_AT.id(), Long.class));
+ stats.set(
+ Column.LAST_UPDATED_SNAPSHOT_ID.id(),
+ record.get(Column.LAST_UPDATED_SNAPSHOT_ID.id(), Long.class));
+
+ return PartitionStatsRecord.create(record.struct(), stats);
+ }
+
+ @VisibleForTesting
+ static Iterator<PartitionStatsRecord> statsToRecords(
Review Comment:
We are doing a lot of logic here that wouldn't be needed with internal readers and writers. Let's at least estimate the amount of work to get the internal writer for Avro, to begin with. Any thoughts, @rdblue?
##########
core/src/main/java/org/apache/iceberg/PartitionStats.java:
##########
@@ -205,6 +211,8 @@ public <T> T get(int pos, Class<T> javaClass) {
public <T> void set(int pos, T value) {
switch (pos) {
case 0:
+ Preconditions.checkArgument(
Review Comment:
It feels a bit awkward to rely on `Record` for a nested field while the main object is simply `StructLike`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1777912510
##########
core/src/main/java/org/apache/iceberg/PartitionStats.java:
##########
@@ -249,4 +250,45 @@ public <T> void set(int pos, T value) {
throw new UnsupportedOperationException("Unknown position: " + pos);
}
}
+
+ @Override
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ public boolean equals(Object other) {
Review Comment:
`StructLikeMap` was previously handling this implicitly. But when `PartitionStatsRecord` wraps `PartitionStats` now for the writers, it needs to override `equals` and `hashcode`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1848581399
##########
core/src/main/java/org/apache/iceberg/PartitionStats.java:
##########
@@ -249,4 +250,45 @@ public <T> void set(int pos, T value) {
throw new UnsupportedOperationException("Unknown position: " + pos);
}
}
+
+ @Override
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ } else if (!(other instanceof PartitionStats)) {
+ return false;
+ }
+
+ PartitionStats that = (PartitionStats) other;
+ return Objects.equals(partition, that.partition)
Review Comment:
Agree. I have added the assert (preconditions) to make sure it is always of the type Record. Also, added a comment that why keeping the type as StructType instead of Record when it is always a record.
It is because in future when we introduce internal parquet writers that works with StructLike instead of Record, we don't have to change method signatures and it will be compatible.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "jbonofre (via GitHub)" <gi...@apache.org>.
jbonofre commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2535341659
@danielcweeks @RussellSpitzer @aokolnychyi would you have some time to take a look on this PR and my proposal (previous comment) ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2613726289
@aokolnychyi, @rdblue, @RussellSpitzer: I have worked on Internal writers, readers for Avro, parquet and PRs got merged.
I have rebased this PR to use the internal writers and readers.
So, this PR is very simple now (no converter logic) and it just writes stats to a file.
I think if we get a good review support it can be merged for 1.8.0 itself. Please take a look.
It was already reviewed before internal writers. So, I don't think much effort is needed. Thanks in advance.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1916789379
##########
core/src/main/java/org/apache/iceberg/PartitionStatsUtil.java:
##########
@@ -133,4 +134,27 @@ private static Collection<PartitionStats> mergeStats(
return statsMap.values();
}
+
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ public static boolean isEqual(
+ Comparator<StructLike> partitionComparator, PartitionStats stats1, PartitionStats stats2) {
Review Comment:
Cannot have `Equals` and `HashCode` for `PartitionStats` class as `StructLike` need to have comparator for `equals()` which forces that class to hold some more things. Setting comparator while serializing and deserializing
that class will be a mess.
Hence, added this util method. Currently used only by tests. But can be useful for developers. So, kept as a util.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2594954262
> #11919
hi @ajantha-bhat, i need to include https://github.com/apache/iceberg/pull/11919, anything else?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1777912510
##########
core/src/main/java/org/apache/iceberg/PartitionStats.java:
##########
@@ -249,4 +250,45 @@ public <T> void set(int pos, T value) {
throw new UnsupportedOperationException("Unknown position: " + pos);
}
}
+
+ @Override
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ public boolean equals(Object other) {
Review Comment:
`StructLikeMap` was previously handling this implicitly. But when `PartitionStatsRecord` wraps `PartitionStats` now for the writers, it needs to override `equals` and `hashcode`
##########
core/src/main/java/org/apache/iceberg/PartitionStatsUtil.java:
##########
@@ -87,13 +90,10 @@ private static PartitionMap<PartitionStats> collectStats(
PartitionMap<PartitionStats> statsMap = PartitionMap.create(table.specs());
int specId = manifest.partitionSpecId();
PartitionSpec spec = table.specs().get(specId);
- PartitionData keyTemplate = new PartitionData(partitionType);
for (ManifestEntry<?> entry : reader.entries()) {
ContentFile<?> file = entry.file();
- StructLike coercedPartition =
- PartitionUtil.coercePartition(partitionType, spec, file.partition());
- StructLike key = keyTemplate.copyFor(coercedPartition);
+ Record key = coercedPartitionRecord(file, spec, partitionType);
Review Comment:
Need `Record` instead of `PartitionData` for the writers.
Cannot keep this conversion in the data module as it just to wraps the same `PartitionStats` object.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "jbonofre (via GitHub)" <gi...@apache.org>.
jbonofre commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2682520892
I did the review and it looks good to me. Thanks !
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "jbonofre (via GitHub)" <gi...@apache.org>.
jbonofre commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1970108920
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a combined partition type which considers
+ * all specs in a table.
+ *
+ * @param unifiedPartitionType unified partition schema type. Could be calculated by {@link
+ * Partitioning#partitionType(Table)}.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType unifiedPartitionType) {
+ Preconditions.checkState(!unifiedPartitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), unifiedPartitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, SnapshotRef.MAIN_BRANCH);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null || branch.equals(SnapshotRef.MAIN_BRANCH),
+ "Couldn't find the snapshot for the branch %s",
+ branch);
+ return null;
+ }
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterable<PartitionStats> records)
+ throws IOException {
+ FileFormat fileFormat =
+ FileFormat.fromString(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+
+ OutputFile outputFile = newPartitionStatsFile(table, fileFormat, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile, fileFormat)) {
+ records.iterator().forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static OutputFile newPartitionStatsFile(
+ Table table, FileFormat fileFormat, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(
+ Locale.ROOT, "partition-stats-%d-%s", snapshotId, UUID.randomUUID()))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(
+ Schema dataSchema, OutputFile outputFile, FileFormat fileFormat) throws IOException {
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(InternalWriter::create)
+ .overwrite()
+ .withSpec(PartitionSpec.unpartitioned())
+ .build();
+ case AVRO:
+ return Avro.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(org.apache.iceberg.avro.InternalWriter::create)
+ .overwrite()
Review Comment:
I agree to keep it aligned with other writers pattern.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2687054193
@RussellSpitzer, @aokolnychyi: Anymore comments? others have approved the changes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1968046827
##########
core/src/test/java/org/apache/iceberg/TestTables.java:
##########
@@ -79,17 +79,7 @@ public static TestTable create(
PartitionSpec spec,
SortOrder sortOrder,
int formatVersion) {
- TestTableOperations ops = new TestTableOperations(name, temp);
- if (ops.current() != null) {
- throw new AlreadyExistsException("Table %s already exists at location: %s", name, temp);
- }
-
- ops.commit(
- null,
- newTableMetadata(
- schema, spec, sortOrder, temp.toString(), ImmutableMap.of(), formatVersion));
-
- return new TestTable(ops, name);
+ return createTable(temp, name, schema, spec, formatVersion, ImmutableMap.of(), sortOrder, null);
Review Comment:
This function came from latest rebase. Someone added it as they wanted to pass properties.
Updated it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1998236833
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
Review Comment:
https://github.com/apache/iceberg/pull/12550
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1961771559
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
Review Comment:
ACK. java doc is already updated
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1940509287
##########
core/src/main/java/org/apache/iceberg/PartitionStatsUtil.java:
##########
@@ -133,4 +134,27 @@ private static Collection<PartitionStats> mergeStats(
return statsMap.values();
}
+
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ public static boolean isEqual(
+ Comparator<StructLike> partitionComparator, PartitionStats stats1, PartitionStats stats2) {
Review Comment:
ok. Can move to test. This is not a big code. Users can replicate it in their environment if required. Always safe to have the scope to minimum.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1940541813
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(Schema dataSchema, OutputFile outputFile)
+ throws IOException {
+ FileFormat fileFormat = fileFormat(outputFile.location());
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(InternalWriter::create)
+ .overwrite()
+ .withSpec(PartitionSpec.unpartitioned())
+ .build();
+ case AVRO:
+ return Avro.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(org.apache.iceberg.avro.InternalWriter::create)
+ .overwrite()
+ .withSpec(PartitionSpec.unpartitioned())
+ .build();
+ case ORC:
+ // Internal writers are not supported for ORC yet.
Review Comment:
If people are intersted to contribute.
Last I discussed it with Ryan, Community is expecting the ORC users to contribute here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1940542661
##########
data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java:
##########
@@ -0,0 +1,582 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.data.PartitionStatsHandler.Column;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionData;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assumptions;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestPartitionStatsHandler {
+ private static final Schema SCHEMA =
+ new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get()));
+
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("c2").identity("c3").build();
+
+ @TempDir public File temp;
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameters(name = "fileFormat = {0}")
+ public static List<Object> parameters() {
+ return Arrays.asList(FileFormat.PARQUET, FileFormat.ORC, FileFormat.AVRO);
+ }
+
+ @Parameter private FileFormat format;
+
+ @Test
+ public void testPartitionStatsOnEmptyTable() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_table"), "empty_table", SCHEMA, SPEC, 2);
Review Comment:
It is similar to other existing tests. @TempDir annottaion should clean up the folders.
##########
data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java:
##########
@@ -0,0 +1,582 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.data.PartitionStatsHandler.Column;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionData;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assumptions;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestPartitionStatsHandler {
+ private static final Schema SCHEMA =
+ new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get()));
+
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("c2").identity("c3").build();
+
+ @TempDir public File temp;
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameters(name = "fileFormat = {0}")
+ public static List<Object> parameters() {
+ return Arrays.asList(FileFormat.PARQUET, FileFormat.ORC, FileFormat.AVRO);
+ }
+
+ @Parameter private FileFormat format;
+
+ @Test
+ public void testPartitionStatsOnEmptyTable() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_table"), "empty_table", SCHEMA, SPEC, 2);
Review Comment:
It is similar to other existing tests. @TempDir annotaion should clean up the folders.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1939029933
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
Review Comment:
Why did we decide to convert an IOException to an unchecked exception?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "RussellSpitzer (via GitHub)" <gi...@apache.org>.
RussellSpitzer commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1816936025
##########
core/src/main/java/org/apache/iceberg/PartitionStats.java:
##########
@@ -249,4 +250,45 @@ public <T> void set(int pos, T value) {
throw new UnsupportedOperationException("Unknown position: " + pos);
}
}
+
+ @Override
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ } else if (!(other instanceof PartitionStats)) {
+ return false;
+ }
+
+ PartitionStats that = (PartitionStats) other;
+ return Objects.equals(partition, that.partition)
+ && specId == that.specId
+ && dataRecordCount == that.dataRecordCount
+ && dataFileCount == that.dataFileCount
+ && totalDataFileSizeInBytes == that.totalDataFileSizeInBytes
+ && positionDeleteRecordCount == that.positionDeleteRecordCount
+ && positionDeleteFileCount == that.positionDeleteFileCount
+ && equalityDeleteRecordCount == that.equalityDeleteRecordCount
+ && equalityDeleteFileCount == that.equalityDeleteFileCount
+ && totalRecordCount == that.totalRecordCount
+ && Objects.equals(lastUpdatedAt, that.lastUpdatedAt)
+ && Objects.equals(lastUpdatedSnapshotId, that.lastUpdatedSnapshotId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ partition,
Review Comment:
Similar comment to above, StructLike doesn't have hash so we'll need to use JavaHashes.struct(...).hash(partition)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1993383799
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
Review Comment:
I have added it for two reasons,
a) I didn't want to use hardcoded strings or string constant for each field name, field id while preparing `NestedField`, hence using the name from enum.
b) Plus I thought enum is cleaner than multiple nested fields constants.
Do you see any drawback for using this enum in terms of readability or performance?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1935881158
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(Schema dataSchema, OutputFile outputFile)
+ throws IOException {
+ FileFormat fileFormat = fileFormat(outputFile.location());
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(InternalWriter::create)
+ .overwrite()
+ .withSpec(PartitionSpec.unpartitioned())
+ .build();
+ case AVRO:
+ return Avro.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(org.apache.iceberg.avro.InternalWriter::create)
Review Comment:
could we add an import?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1960061400
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
Review Comment:
Be aware of the current limitation (#12300) which is being fixed (#12301)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1961364417
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
Review Comment:
Let's move on with returning `null` then, but we need to highlight that in the javadoc
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1961765409
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
Review Comment:
Yeah.. sorry, I have found it and closed it again 😄
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1963130151
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a combined partition type which considers
+ * all specs in a table.
+ *
+ * @param unifiedPartitionType unified partition schema type. Could be calculated by {@link
+ * Partitioning#partitionType(Table)}.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType unifiedPartitionType) {
+ Preconditions.checkState(!unifiedPartitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), unifiedPartitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, SnapshotRef.MAIN_BRANCH);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null || branch.equals(SnapshotRef.MAIN_BRANCH),
+ "Couldn't find the snapshot for the branch %s",
+ branch);
+ return null;
+ }
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterable<PartitionStats> records)
+ throws IOException {
+ FileFormat fileFormat =
+ FileFormat.fromString(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+
+ OutputFile outputFile = newPartitionStatsFile(table, fileFormat, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile, fileFormat)) {
+ records.iterator().forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileName) {
+ FileFormat format = FileFormat.fromFileName(fileName);
+ Preconditions.checkArgument(format != null, "Unable to determine format of file: %s", fileName);
+ return format;
+ }
Review Comment:
Does it worth to have a method for this? Only used in one place and this is basically a one-liner with a check.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1961373018
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records)
+ throws IOException {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile)) {
+ records.forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(Schema dataSchema, OutputFile outputFile)
+ throws IOException {
+ FileFormat fileFormat = fileFormat(outputFile.location());
Review Comment:
Please pass down the arguments. I still think that getting the file format from the filename is bad practice, and it is especially bad when we have the file format at hand.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2378257408
@aokolnychyi: This PR is ready. But as we discussed previously, this PR wraps the `PartitionStats` into a `Record` as the writers cannot work with Iceberg internal objects yet.
I will explore adding the internal writers for Parquet and Orc. Similar to https://github.com/apache/iceberg/pull/11108.
If we fail to have it ready by 1.7.0, I think it makes sense to merge this PR and introduce the optimized writer in the next version by deprecating this writer.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1937114400
##########
data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java:
##########
@@ -0,0 +1,582 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.data.PartitionStatsHandler.Column;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionData;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assumptions;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestPartitionStatsHandler {
+ private static final Schema SCHEMA =
+ new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get()));
+
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("c2").identity("c3").build();
+
+ @TempDir public File temp;
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameters(name = "fileFormat = {0}")
+ public static List<Object> parameters() {
+ return Arrays.asList(FileFormat.PARQUET, FileFormat.ORC, FileFormat.AVRO);
+ }
+
+ @Parameter private FileFormat format;
+
+ @Test
+ public void testPartitionStatsOnEmptyTable() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_table"), "empty_table", SCHEMA, SPEC, 2);
+ assertThat(PartitionStatsHandler.computeAndWriteStatsFile(testTable)).isNull();
+ }
+
+ @Test
+ public void testPartitionStatsOnEmptyBranch() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_branch"), "empty_branch", SCHEMA, SPEC, 2);
+ testTable.manageSnapshots().createBranch("b1").commit();
+ PartitionStatisticsFile partitionStatisticsFile =
+ PartitionStatsHandler.computeAndWriteStatsFile(testTable, "b1");
+ // creates an empty stats file since the dummy snapshot exist
+ assertThat(partitionStatisticsFile.fileSizeInBytes()).isEqualTo(0L);
Review Comment:
But it won't be flaky. So, we can update the test if the behavior changes. This is as per current behavior.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1939043730
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(Schema dataSchema, OutputFile outputFile)
+ throws IOException {
+ FileFormat fileFormat = fileFormat(outputFile.location());
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(InternalWriter::create)
+ .overwrite()
+ .withSpec(PartitionSpec.unpartitioned())
+ .build();
+ case AVRO:
+ return Avro.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(org.apache.iceberg.avro.InternalWriter::create)
+ .overwrite()
+ .withSpec(PartitionSpec.unpartitioned())
+ .build();
+ case ORC:
+ // Internal writers are not supported for ORC yet.
Review Comment:
Do we plan to support ORC Internal writers?
Or do we plan to support partition statistics file for ORC?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1939025938
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
Review Comment:
Is this for handling an empty table?
How users of this method will use the returned null value?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1940510235
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
Review Comment:
Utility classes (with private constructor) ideally preferred to be final. I can remove it if not a requirement in this project.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1941240868
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
Review Comment:
I'm not sure what is the correct answer here. That's why I was asking "How users of this method will use the returned null value?"
Pros for `null`:
- Easy to implement
- No extra file is written
Pros for `empty`:
- Easier to use
- More near to the real life expectations a table without a snapshot is still a valid table, just with 0 records
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2432771734
@RussellSpitzer: It would be good to have this in 1.7.0.
I am waiting from a month for a review.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1853193408
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.BiFunction;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.data.avro.DataReader;
+import org.apache.iceberg.data.orc.GenericOrcReader;
+import org.apache.iceberg.data.parquet.GenericParquetReaders;
+import org.apache.iceberg.encryption.EncryptedFiles;
+import org.apache.iceberg.encryption.EncryptionKeyMetadata;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.FileWriterFactory;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.orc.ORC;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Schema schema = schema(partitionType);
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ Iterator<PartitionStatsRecord> convertedRecords = statsToRecords(sortedStats, schema);
+ return writePartitionStatsFile(table, currentSnapshot.snapshotId(), schema, convertedRecords);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStatsRecord> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+ FileWriterFactory<Record> factory =
+ GenericFileWriterFactory.builderFor(table)
+ .dataSchema(dataSchema)
+ .dataFileFormat(fileFormat(outputFile.location()))
+ .build();
+ DataWriter<Record> writer =
+ factory.newDataWriter(
+ EncryptedFiles.encryptedOutput(outputFile, EncryptionKeyMetadata.EMPTY),
+ PartitionSpec.unpartitioned(),
+ null);
+ try (Closeable toClose = writer) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStatsRecord> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<Record> records;
+ FileFormat fileFormat = fileFormat(inputFile.location());
+ switch (fileFormat) {
+ case PARQUET:
+ records =
+ Parquet.read(inputFile)
+ .project(schema)
+ .createReaderFunc(
+ fileSchema -> GenericParquetReaders.buildReader(schema, fileSchema))
+ .build();
+ break;
+ case ORC:
+ records =
+ ORC.read(inputFile)
+ .project(schema)
+ .createReaderFunc(fileSchema -> GenericOrcReader.buildReader(schema, fileSchema))
+ .build();
+ break;
+ case AVRO:
+ records = Avro.read(inputFile).project(schema).createReaderFunc(DataReader::create).build();
+ break;
+ default:
+ throw new UnsupportedOperationException("Unsupported file format:" + fileFormat.name());
+ }
+
+ return CloseableIterable.transform(
+ records, PartitionStatsHandler::recordToPartitionStatsRecord);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static PartitionStatsRecord recordToPartitionStatsRecord(Record record) {
+ PartitionStats stats =
+ new PartitionStats(
+ record.get(Column.PARTITION.id(), StructLike.class),
+ record.get(Column.SPEC_ID.id(), Integer.class));
+ stats.set(Column.DATA_RECORD_COUNT.id(), record.get(Column.DATA_RECORD_COUNT.id(), Long.class));
+ stats.set(Column.DATA_FILE_COUNT.id(), record.get(Column.DATA_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(),
+ record.get(Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(), Long.class));
+ stats.set(
+ Column.POSITION_DELETE_RECORD_COUNT.id(),
+ record.get(Column.POSITION_DELETE_RECORD_COUNT.id(), Long.class));
+ stats.set(
+ Column.POSITION_DELETE_FILE_COUNT.id(),
+ record.get(Column.POSITION_DELETE_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.EQUALITY_DELETE_RECORD_COUNT.id(),
+ record.get(Column.EQUALITY_DELETE_RECORD_COUNT.id(), Long.class));
+ stats.set(
+ Column.EQUALITY_DELETE_FILE_COUNT.id(),
+ record.get(Column.EQUALITY_DELETE_FILE_COUNT.id(), Integer.class));
+ stats.set(
+ Column.TOTAL_RECORD_COUNT.id(), record.get(Column.TOTAL_RECORD_COUNT.id(), Long.class));
+ stats.set(Column.LAST_UPDATED_AT.id(), record.get(Column.LAST_UPDATED_AT.id(), Long.class));
+ stats.set(
+ Column.LAST_UPDATED_SNAPSHOT_ID.id(),
+ record.get(Column.LAST_UPDATED_SNAPSHOT_ID.id(), Long.class));
+
+ return PartitionStatsRecord.create(record.struct(), stats);
+ }
+
+ @VisibleForTesting
+ static Iterator<PartitionStatsRecord> statsToRecords(
Review Comment:
@rdblue, @aokolnychyi, @RussellSpitzer: Can we please conclude on this?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1968045379
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a combined partition type which considers
+ * all specs in a table.
+ *
+ * @param unifiedPartitionType unified partition schema type. Could be calculated by {@link
+ * Partitioning#partitionType(Table)}.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType unifiedPartitionType) {
+ Preconditions.checkState(!unifiedPartitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), unifiedPartitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, SnapshotRef.MAIN_BRANCH);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null || branch.equals(SnapshotRef.MAIN_BRANCH),
+ "Couldn't find the snapshot for the branch %s",
+ branch);
+ return null;
+ }
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterable<PartitionStats> records)
+ throws IOException {
+ FileFormat fileFormat =
+ FileFormat.fromString(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+
+ OutputFile outputFile = newPartitionStatsFile(table, fileFormat, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile, fileFormat)) {
+ records.iterator().forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static OutputFile newPartitionStatsFile(
+ Table table, FileFormat fileFormat, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(
+ Locale.ROOT, "partition-stats-%d-%s", snapshotId, UUID.randomUUID()))));
Review Comment:
Currently it is same as how other stats are supported (puffin files). I saw your reply on mailing list. Lets see what others think. We can normalize that behavior for all stats. No need to wait or block this PR for that.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "RussellSpitzer (via GitHub)" <gi...@apache.org>.
RussellSpitzer commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1816957914
##########
data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java:
##########
@@ -0,0 +1,569 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.data.PartitionStatsHandler.Column;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.UUIDUtil;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestPartitionStatsHandler {
+ private static final Schema SCHEMA =
+ new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get()));
+
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("c2").identity("c3").build();
+
+ @TempDir public File temp;
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameters(name = "fileFormat = {0}")
+ public static List<Object> parameters() {
+ return Arrays.asList(FileFormat.PARQUET, FileFormat.ORC, FileFormat.AVRO);
+ }
+
+ @Parameter private FileFormat format;
+
+ @Test
+ public void testPartitionStatsOnEmptyTable() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_table"), "empty_table", SCHEMA, SPEC, 2);
+ assertThat(PartitionStatsHandler.computeAndWriteStatsFile(testTable)).isNull();
+ }
+
+ @Test
+ public void testPartitionStatsOnEmptyBranch() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_branch"), "empty_branch", SCHEMA, SPEC, 2);
+ testTable.manageSnapshots().createBranch("b1").commit();
+ PartitionStatisticsFile partitionStatisticsFile =
+ PartitionStatsHandler.computeAndWriteStatsFile(testTable, "b1");
+ // creates an empty stats file since the dummy snapshot exist
+ assertThat(partitionStatisticsFile.fileSizeInBytes()).isEqualTo(0L);
+ assertThat(partitionStatisticsFile.snapshotId())
+ .isEqualTo(testTable.refs().get("b1").snapshotId());
+ }
+
+ @Test
+ public void testPartitionStatsOnInvalidSnapshot() throws Exception {
+ Table testTable =
+ TestTables.create(tempDir("invalid_snapshot"), "invalid_snapshot", SCHEMA, SPEC, 2);
+ assertThatThrownBy(
+ () -> PartitionStatsHandler.computeAndWriteStatsFile(testTable, "INVALID_BRANCH"))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Couldn't find the snapshot for the branch INVALID_BRANCH");
+ }
+
+ @Test
+ public void testPartitionStatsOnUnPartitionedTable() throws Exception {
+ Table testTable =
+ TestTables.create(
+ tempDir("unpartitioned_table"),
+ "unpartitioned_table",
+ SCHEMA,
+ PartitionSpec.unpartitioned(),
+ 2);
+
+ List<Record> records = prepareRecords(testTable.schema());
+ DataFile dataFile = FileHelpers.writeDataFile(testTable, outputFile(), records);
+ testTable.newAppend().appendFile(dataFile).commit();
+
+ assertThatThrownBy(() -> PartitionStatsHandler.computeAndWriteStatsFile(testTable))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage("table must be partitioned");
+ }
+
+ @Test
+ public void testAllDatatypePartitionWriting() throws Exception {
+ Schema schema =
+ new Schema(
+ required(100, "id", Types.LongType.get()),
+ optional(101, "data", Types.StringType.get()),
+ required(102, "b", Types.BooleanType.get()),
+ optional(103, "i", Types.IntegerType.get()),
+ required(104, "l", Types.LongType.get()),
+ optional(105, "f", Types.FloatType.get()),
+ required(106, "d", Types.DoubleType.get()),
+ optional(107, "date", Types.DateType.get()),
+ required(108, "ts", Types.TimestampType.withoutZone()),
+ required(110, "s", Types.StringType.get()),
+ required(111, "uuid", Types.UUIDType.get()),
+ required(112, "fixed", Types.FixedType.ofLength(7)),
+ optional(113, "bytes", Types.BinaryType.get()),
+ required(114, "dec_9_0", Types.DecimalType.of(9, 0)),
+ required(115, "dec_11_2", Types.DecimalType.of(11, 2)),
+ required(116, "dec_38_10", Types.DecimalType.of(38, 10)), // maximum precision
+ required(117, "time", Types.TimeType.get()));
+
+ PartitionSpec spec =
+ PartitionSpec.builderFor(schema)
+ .identity("b")
+ .identity("i")
+ .identity("l")
+ .identity("f")
+ .identity("d")
+ .identity("date")
+ .identity("ts")
+ .identity("s")
+ .identity("uuid")
+ .identity("fixed")
+ .identity("bytes")
+ .identity("dec_9_0")
+ .identity("dec_11_2")
+ .identity("dec_38_10")
+ .identity("time")
+ .build();
+
+ Table testTable =
+ TestTables.create(
+ tempDir("test_all_type"), "test_all_type", schema, spec, SortOrder.unsorted(), 2);
+
+ Types.StructType partitionSchema = Partitioning.partitionType(testTable);
+ Schema dataSchema = PartitionStatsHandler.schema(partitionSchema);
+
+ Record partitionData =
+ GenericRecord.create(dataSchema.findField(Column.PARTITION.name()).type().asStructType());
+ partitionData.set(0, true);
+ partitionData.set(1, 42);
+ partitionData.set(2, 42L);
+ partitionData.set(3, 3.14f);
+ partitionData.set(4, 3.141592653589793);
+ partitionData.set(5, Literal.of("2022-01-01").to(Types.DateType.get()).value());
+ partitionData.set(
+ 6, Literal.of("2017-12-01T10:12:55.038194").to(Types.TimestampType.withoutZone()).value());
+ partitionData.set(7, "string");
+ partitionData.set(8, UUIDUtil.convertToByteBuffer(UUID.randomUUID()));
+ partitionData.set(9, new byte[] {0, 1, 2, 3, 4, 5, 6});
+ partitionData.set(10, new byte[] {1, 2, 3});
+ partitionData.set(11, Literal.of("123456789").to(Types.DecimalType.of(9, 0)).value());
+ partitionData.set(12, Literal.of("1234567.89").to(Types.DecimalType.of(11, 2)).value());
+ partitionData.set(
+ 13, Literal.of("12345678901234567890.1234567890").to(Types.DecimalType.of(38, 10)).value());
+ partitionData.set(14, Literal.of("10:10:10").to(Types.TimeType.get()).value());
+
+ PartitionStats partitionStats = new PartitionStats(partitionData, RANDOM.nextInt(10));
+ partitionStats.set(Column.DATA_RECORD_COUNT.id(), RANDOM.nextLong());
+ partitionStats.set(Column.DATA_FILE_COUNT.id(), RANDOM.nextInt());
+ partitionStats.set(Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.id(), 1024L * RANDOM.nextInt(20));
+
+ Iterator<PartitionStatsRecord> convertedRecords =
+ PartitionStatsHandler.statsToRecords(Collections.singletonList(partitionStats), dataSchema);
Review Comment:
Can we add a few more records here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1777914781
##########
core/src/main/java/org/apache/iceberg/data/PartitionStatsRecord.java:
##########
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.relocated.com.google.common.base.Objects;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.types.Types.StructType;
+
+/** Wraps the {@link PartitionStats} as {@link Record}. Used by generic writers and readers. */
+public class PartitionStatsRecord implements Record, StructLike {
+ private static final LoadingCache<StructType, Map<String, Integer>> NAME_MAP_CACHE =
Review Comment:
Class is similar to `GenericRecord` but for a specific partition stats schema.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1777914012
##########
core/src/main/java/org/apache/iceberg/PartitionStatsUtil.java:
##########
@@ -87,13 +90,10 @@ private static PartitionMap<PartitionStats> collectStats(
PartitionMap<PartitionStats> statsMap = PartitionMap.create(table.specs());
int specId = manifest.partitionSpecId();
PartitionSpec spec = table.specs().get(specId);
- PartitionData keyTemplate = new PartitionData(partitionType);
for (ManifestEntry<?> entry : reader.entries()) {
ContentFile<?> file = entry.file();
- StructLike coercedPartition =
- PartitionUtil.coercePartition(partitionType, spec, file.partition());
- StructLike key = keyTemplate.copyFor(coercedPartition);
+ Record key = coercedPartitionRecord(file, spec, partitionType);
Review Comment:
Need `Record` instead of `PartitionData` for the writers.
Cannot keep this conversion in the data module as it just to wraps the same `PartitionStats` object.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1993323183
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a combined partition type which considers
+ * all specs in a table.
+ *
+ * @param unifiedPartitionType unified partition schema type. Could be calculated by {@link
+ * Partitioning#partitionType(Table)}.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType unifiedPartitionType) {
+ Preconditions.checkState(!unifiedPartitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), unifiedPartitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, SnapshotRef.MAIN_BRANCH);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null || branch.equals(SnapshotRef.MAIN_BRANCH),
+ "Couldn't find the snapshot for the branch %s",
+ branch);
+ return null;
+ }
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterable<PartitionStats> records)
+ throws IOException {
+ FileFormat fileFormat =
+ FileFormat.fromString(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+
+ OutputFile outputFile = newPartitionStatsFile(table, fileFormat, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile, fileFormat)) {
+ records.iterator().forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static OutputFile newPartitionStatsFile(
+ Table table, FileFormat fileFormat, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(
+ Locale.ROOT, "partition-stats-%d-%s", snapshotId, UUID.randomUUID()))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(
+ Schema dataSchema, OutputFile outputFile, FileFormat fileFormat) throws IOException {
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(InternalWriter::create)
+ .withSpec(PartitionSpec.unpartitioned())
+ .build();
+ case AVRO:
+ return Avro.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(org.apache.iceberg.avro.InternalWriter::create)
+ .withSpec(PartitionSpec.unpartitioned())
+ .build();
+ case ORC:
+ // Internal writers are not supported for ORC yet.
+ default:
+ throw new UnsupportedOperationException("Unsupported file format:" + fileFormat.name());
+ }
+ }
+
+ private static CloseableIterable<StructLike> dataReader(Schema schema, InputFile inputFile) {
+ FileFormat fileFormat = FileFormat.fromFileName(inputFile.location());
+ Preconditions.checkArgument(
+ fileFormat != null, "Unable to determine format of file: %s", inputFile.location());
+
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.read(inputFile)
+ .project(schema)
+ .createReaderFunc(
+ fileSchema ->
+ org.apache.iceberg.data.parquet.InternalReader.create(schema, fileSchema))
+ .build();
+ case AVRO:
+ return Avro.read(inputFile)
+ .project(schema)
+ .createReaderFunc(fileSchema -> InternalReader.create(schema))
+ .build();
+ case ORC:
+ // Internal readers are not supported for ORC yet.
+ default:
+ throw new UnsupportedOperationException("Unsupported file format:" + fileFormat.name());
+ }
+ }
+
+ private static PartitionStats recordToPartitionStats(StructLike record) {
+ PartitionStats stats =
Review Comment:
> CloseableIterable<StructLike> returned by the data reader reuses the same object? If so, we don't have to create a new PartitionStats object for every record
By default it doesn't. While adding the internal readers, I have added the support for `reuseContainers` too. So, we can enable it.
I am not sure about enabling it here as it is an end user API, if the user want to keep stats for more than one partition at once, like our testcase where we prepare a list from this iterable (instead of consuming one by one), we cannot reuse the container. Hence, I didn't enable it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1998041969
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
Review Comment:
Thx!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2580006220
> @deniskuzZ: Thanks for testing out. We are working on Internal parquet/Avro/orc readers and writes. partition stats will use them. So, we don't need to go through these converters. I will retest all the data types once I use internal writers for partition stats.
>
> PRs: #11919 #11904
JFYI, once I removed `IdentityPartitionConverters.convertConstant` from the `statsToRecords->convertPartitionValues` issue with `partitionToPath` method is gone.
IDK, maybe you had some other idea of how to do the partition filtering on the returned stats, but I used the following
````
spec.partitionToPath(stats.unwrap().partition())
.equals(partish.getPartition().getName())
````
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1935881158
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(Schema dataSchema, OutputFile outputFile)
+ throws IOException {
+ FileFormat fileFormat = fileFormat(outputFile.location());
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(InternalWriter::create)
+ .overwrite()
+ .withSpec(PartitionSpec.unpartitioned())
+ .build();
+ case AVRO:
+ return Avro.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(org.apache.iceberg.avro.InternalWriter::create)
Review Comment:
could we add an import?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1777915925
##########
core/src/test/java/org/apache/iceberg/TestPartitionStatsUtil.java:
##########
@@ -370,17 +371,17 @@ public void testPartitionStatsWithSchemaEvolution() throws Exception {
snapshot2.snapshotId()));
}
- private static PartitionData partitionData(Types.StructType partitionType, String c2, String c3) {
- PartitionData partitionData = new PartitionData(partitionType);
- partitionData.set(0, c2);
- partitionData.set(1, c3);
- return partitionData;
+ private static StructLike partitionData(Types.StructType partitionType, String c2, String c3) {
+ GenericRecord record = GenericRecord.create(partitionType);
Review Comment:
Stores the `Record` instead of `PartitionData` in the core logic. So, testcase is updated to expect `Record`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1937236363
##########
data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java:
##########
@@ -0,0 +1,582 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.data.PartitionStatsHandler.Column;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionData;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assumptions;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestPartitionStatsHandler {
+ private static final Schema SCHEMA =
+ new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get()));
+
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("c2").identity("c3").build();
+
+ @TempDir public File temp;
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameters(name = "fileFormat = {0}")
+ public static List<Object> parameters() {
+ return Arrays.asList(FileFormat.PARQUET, FileFormat.ORC, FileFormat.AVRO);
+ }
+
+ @Parameter private FileFormat format;
+
+ @Test
+ public void testPartitionStatsOnEmptyTable() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_table"), "empty_table", SCHEMA, SPEC, 2);
+ assertThat(PartitionStatsHandler.computeAndWriteStatsFile(testTable)).isNull();
+ }
+
+ @Test
+ public void testPartitionStatsOnEmptyBranch() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_branch"), "empty_branch", SCHEMA, SPEC, 2);
+ testTable.manageSnapshots().createBranch("b1").commit();
+ PartitionStatisticsFile partitionStatisticsFile =
+ PartitionStatsHandler.computeAndWriteStatsFile(testTable, "b1");
+ // creates an empty stats file since the dummy snapshot exist
+ assertThat(partitionStatisticsFile.fileSizeInBytes()).isEqualTo(0L);
Review Comment:
Thanks for your reviews. I hope we ship this feature soon and glad to know Hive, Trino are waiting for this feature.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1939015974
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
Review Comment:
nit: Start the error message with capital letter
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1939049410
##########
data/src/test/java/org/apache/iceberg/data/TestPartitionStatsHandler.java:
##########
@@ -0,0 +1,582 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.data.PartitionStatsHandler.Column;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionData;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assumptions;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestPartitionStatsHandler {
+ private static final Schema SCHEMA =
+ new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get()));
+
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("c2").identity("c3").build();
+
+ @TempDir public File temp;
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameters(name = "fileFormat = {0}")
+ public static List<Object> parameters() {
+ return Arrays.asList(FileFormat.PARQUET, FileFormat.ORC, FileFormat.AVRO);
+ }
+
+ @Parameter private FileFormat format;
+
+ @Test
+ public void testPartitionStatsOnEmptyTable() throws Exception {
+ Table testTable = TestTables.create(tempDir("empty_table"), "empty_table", SCHEMA, SPEC, 2);
Review Comment:
Is this table cleaned up after the test?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1929434675
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.BiFunction;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.data.avro.DataReader;
+import org.apache.iceberg.data.orc.GenericOrcReader;
+import org.apache.iceberg.data.parquet.GenericParquetReaders;
+import org.apache.iceberg.encryption.EncryptedFiles;
+import org.apache.iceberg.encryption.EncryptionKeyMetadata;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.FileWriterFactory;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.orc.ORC;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
Review Comment:
> What if we introduce the write method that gets the stats iterator as an argument.
We do have a package-private write method for that `writePartitionStatsFile`, just have to make it public.
Didn't do because what if the users don't sort the stats before, it will violate the spec.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1941197057
##########
core/src/main/java/org/apache/iceberg/PartitionStatsUtil.java:
##########
@@ -133,4 +134,27 @@ private static Collection<PartitionStats> mergeStats(
return statsMap.values();
}
+
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ public static boolean isEqual(
+ Comparator<StructLike> partitionComparator, PartitionStats stats1, PartitionStats stats2) {
Review Comment:
Please move to test
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1963462471
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a combined partition type which considers
+ * all specs in a table.
+ *
+ * @param unifiedPartitionType unified partition schema type. Could be calculated by {@link
+ * Partitioning#partitionType(Table)}.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType unifiedPartitionType) {
+ Preconditions.checkState(!unifiedPartitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), unifiedPartitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, SnapshotRef.MAIN_BRANCH);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null || branch.equals(SnapshotRef.MAIN_BRANCH),
+ "Couldn't find the snapshot for the branch %s",
+ branch);
+ return null;
+ }
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterable<PartitionStats> records)
+ throws IOException {
+ FileFormat fileFormat =
+ FileFormat.fromString(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+
+ OutputFile outputFile = newPartitionStatsFile(table, fileFormat, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile, fileFormat)) {
+ records.iterator().forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileName) {
+ FileFormat format = FileFormat.fromFileName(fileName);
+ Preconditions.checkArgument(format != null, "Unable to determine format of file: %s", fileName);
+ return format;
+ }
Review Comment:
ok. Inlined the function.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "gaborkaszab (via GitHub)" <gi...@apache.org>.
gaborkaszab commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1967808710
##########
core/src/test/java/org/apache/iceberg/TestTables.java:
##########
@@ -79,17 +79,7 @@ public static TestTable create(
PartitionSpec spec,
SortOrder sortOrder,
int formatVersion) {
- TestTableOperations ops = new TestTableOperations(name, temp);
- if (ops.current() != null) {
- throw new AlreadyExistsException("Table %s already exists at location: %s", name, temp);
- }
-
- ops.commit(
- null,
- newTableMetadata(
- schema, spec, sortOrder, temp.toString(), ImmutableMap.of(), formatVersion));
-
- return new TestTable(ops, name);
+ return createTable(temp, name, schema, spec, formatVersion, ImmutableMap.of(), sortOrder, null);
Review Comment:
You moved the implementation of these create functions into a common place that is a nice refactor. However, I see one create() function above at L55 that still has this same implementation body that seems redundant. Can't you also replace that with a function call to the common implementation?
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a combined partition type which considers
+ * all specs in a table.
+ *
+ * @param unifiedPartitionType unified partition schema type. Could be calculated by {@link
+ * Partitioning#partitionType(Table)}.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType unifiedPartitionType) {
+ Preconditions.checkState(!unifiedPartitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), unifiedPartitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, SnapshotRef.MAIN_BRANCH);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null || branch.equals(SnapshotRef.MAIN_BRANCH),
+ "Couldn't find the snapshot for the branch %s",
+ branch);
+ return null;
+ }
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterable<PartitionStats> records)
+ throws IOException {
+ FileFormat fileFormat =
+ FileFormat.fromString(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+
+ OutputFile outputFile = newPartitionStatsFile(table, fileFormat, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile, fileFormat)) {
+ records.iterator().forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static OutputFile newPartitionStatsFile(
+ Table table, FileFormat fileFormat, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(
+ Locale.ROOT, "partition-stats-%d-%s", snapshotId, UUID.randomUUID()))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(
+ Schema dataSchema, OutputFile outputFile, FileFormat fileFormat) throws IOException {
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(InternalWriter::create)
+ .overwrite()
+ .withSpec(PartitionSpec.unpartitioned())
+ .build();
+ case AVRO:
+ return Avro.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(org.apache.iceberg.avro.InternalWriter::create)
+ .overwrite()
Review Comment:
See above: stat file names are expected to be unique, we shouldn't overwrite any existing files.
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a combined partition type which considers
+ * all specs in a table.
+ *
+ * @param unifiedPartitionType unified partition schema type. Could be calculated by {@link
+ * Partitioning#partitionType(Table)}.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType unifiedPartitionType) {
+ Preconditions.checkState(!unifiedPartitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), unifiedPartitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, SnapshotRef.MAIN_BRANCH);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null || branch.equals(SnapshotRef.MAIN_BRANCH),
+ "Couldn't find the snapshot for the branch %s",
+ branch);
+ return null;
+ }
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterable<PartitionStats> records)
+ throws IOException {
+ FileFormat fileFormat =
+ FileFormat.fromString(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+
+ OutputFile outputFile = newPartitionStatsFile(table, fileFormat, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile, fileFormat)) {
+ records.iterator().forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static OutputFile newPartitionStatsFile(
+ Table table, FileFormat fileFormat, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(
+ Locale.ROOT, "partition-stats-%d-%s", snapshotId, UUID.randomUUID()))));
Review Comment:
I think this is the way to follow, however, we should somehow raise awareness that the approach to update both the table stats and partition stats can produce orphan files within the table folder. I think we should keep pinging the ongoing conversation on dev@ about this because with this approach we could easily flood the folder with orphan files and when we drop the table these files will remain.
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records)
+ throws IOException {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile)) {
+ records.forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(Schema dataSchema, OutputFile outputFile)
+ throws IOException {
+ FileFormat fileFormat = fileFormat(outputFile.location());
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(InternalWriter::create)
+ .overwrite()
Review Comment:
Thanks! Shouldn't we also remove the `.overwrite()` call for the writer here? In case we assume the stat files have unique name we shouldn't allow overwriting anything that shares the same name.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1968041114
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a combined partition type which considers
+ * all specs in a table.
+ *
+ * @param unifiedPartitionType unified partition schema type. Could be calculated by {@link
+ * Partitioning#partitionType(Table)}.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType unifiedPartitionType) {
+ Preconditions.checkState(!unifiedPartitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), unifiedPartitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, SnapshotRef.MAIN_BRANCH);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null || branch.equals(SnapshotRef.MAIN_BRANCH),
+ "Couldn't find the snapshot for the branch %s",
+ branch);
+ return null;
+ }
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterable<PartitionStats> records)
+ throws IOException {
+ FileFormat fileFormat =
+ FileFormat.fromString(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+
+ OutputFile outputFile = newPartitionStatsFile(table, fileFormat, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile, fileFormat)) {
+ records.iterator().forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static OutputFile newPartitionStatsFile(
+ Table table, FileFormat fileFormat, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(
+ Locale.ROOT, "partition-stats-%d-%s", snapshotId, UUID.randomUUID()))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(
+ Schema dataSchema, OutputFile outputFile, FileFormat fileFormat) throws IOException {
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(InternalWriter::create)
+ .overwrite()
+ .withSpec(PartitionSpec.unpartitioned())
+ .build();
+ case AVRO:
+ return Avro.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(org.apache.iceberg.avro.InternalWriter::create)
+ .overwrite()
Review Comment:
Followed other writers pattern. Removed it now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1968039303
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
Review Comment:
> I'm wondering if it would make sense to simply return in that function if the file is null now
Agree. Updated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1968046827
##########
core/src/test/java/org/apache/iceberg/TestTables.java:
##########
@@ -79,17 +79,7 @@ public static TestTable create(
PartitionSpec spec,
SortOrder sortOrder,
int formatVersion) {
- TestTableOperations ops = new TestTableOperations(name, temp);
- if (ops.current() != null) {
- throw new AlreadyExistsException("Table %s already exists at location: %s", name, temp);
- }
-
- ops.commit(
- null,
- newTableMetadata(
- schema, spec, sortOrder, temp.toString(), ImmutableMap.of(), formatVersion));
-
- return new TestTable(ops, name);
+ return createTable(temp, name, schema, spec, formatVersion, ImmutableMap.of(), sortOrder, null);
Review Comment:
This function came from latest rebase. Someone added it as they wanted to properties.
Updated it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1944505874
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
Review Comment:
I think these methods will be used by external engine developers who are not that familiar with the internals with Iceberg. At least highlight the requirement in naming as well, and provide a way to gather the information:
```
/**
* Generates the partition stats file schema based on a combined partition type which considers all specs in a
* table. Could be calculated by {@link Partitioning#partitionType(Table)}.
*
* @param partitionType unified partition schema type.
* @return a schema that corresponds to the provided unified partition type.
*/
public static Schema schema(StructType unifiedPartitionType) {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary merged PR #11216:
URL: https://github.com/apache/iceberg/pull/11216
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "aokolnychyi (via GitHub)" <gi...@apache.org>.
aokolnychyi commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2489529147
I had a conversation with @rdblue today about internal writers. Ryan should have a bit of time to help/guide.
I will check the current implementation today too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1818881073
##########
core/src/main/java/org/apache/iceberg/PartitionStats.java:
##########
@@ -249,4 +250,45 @@ public <T> void set(int pos, T value) {
throw new UnsupportedOperationException("Unknown position: " + pos);
}
}
+
+ @Override
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ } else if (!(other instanceof PartitionStats)) {
+ return false;
+ }
+
+ PartitionStats that = (PartitionStats) other;
+ return Objects.equals(partition, that.partition)
Review Comment:
I also tried adding the comparator logic today and pass the comparator of partition type.
Since, we are converting the partition values for the writer in `PartitionStatsHandler.statsToRecords()`.
Comparator expects integer value for date column but we have converted the values to LocalDate, hence comparison fails.
If I don't use the comparator, Record.equals() will be called which does array compare and passes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "RussellSpitzer (via GitHub)" <gi...@apache.org>.
RussellSpitzer commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1819329112
##########
core/src/main/java/org/apache/iceberg/PartitionStats.java:
##########
@@ -249,4 +250,45 @@ public <T> void set(int pos, T value) {
throw new UnsupportedOperationException("Unknown position: " + pos);
}
}
+
+ @Override
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ } else if (!(other instanceof PartitionStats)) {
+ return false;
+ }
+
+ PartitionStats that = (PartitionStats) other;
+ return Objects.equals(partition, that.partition)
Review Comment:
Sorry but we can't assume subtype here unless you want to assert and change the field type above. If we say something is structLike we can't assume it behaves like a record (even if given the current code we know it won't.) If you want it to be a record you need to cast it and assert earlier in the class.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1950036853
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
Review Comment:
`PartitionStatisticsFile` is just an interface.
PartitionStatsHandler is a helper method to build that file along with the writing and reading capabilities. Hence it is kept here.
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
Review Comment:
`PartitionStatisticsFile` is just an interface.
`PartitionStatsHandler` is a helper method to build that file along with the writing and reading capabilities. Hence it is kept here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1950061705
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records)
+ throws IOException {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile)) {
+ records.forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(Schema dataSchema, OutputFile outputFile)
+ throws IOException {
+ FileFormat fileFormat = fileFormat(outputFile.location());
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(InternalWriter::create)
+ .overwrite()
Review Comment:
It is similar to how our current writers are present in `BaseFileWriterFactory`, `GenericAppenderFactory`.
> Assume we already have a stats file and we overwrite it here. Wouldn't it cause any issues if someone is currently reading the file we overwrite now?
Snapshot id will be unique for each new commit. If users are trying to compute for stats for the same snapshot. It is overwriting the file as the contents doesn't change.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2646947773
I was travelling for past four days. Will be taking a look at the new comments tomorrow.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1950004069
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
Review Comment:
> I'm not sure what is the correct answer here
@pvary:
I am also neutral about this argument. I prefer APIs to return null instead of writing empty file.
@gaborkaszab:
I except users of the APIs to handle the extra null check instead of throwing exception here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "gaborkaszab (via GitHub)" <gi...@apache.org>.
gaborkaszab commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1969209511
##########
core/src/main/java/org/apache/iceberg/SetPartitionStatistics.java:
##########
@@ -36,7 +35,10 @@ public SetPartitionStatistics(TableOperations ops) {
@Override
public UpdatePartitionStatistics setPartitionStatistics(PartitionStatisticsFile file) {
- Preconditions.checkArgument(null != file, "partition statistics file must not be null");
+ if (file == null) {
Review Comment:
Shouldn't we update the comment on the interface to be clear that a null param results a noop?
I checked the implementation again for SetStatistics.setStatistics() and apparently there is no special handling for null inputs. I became a bit hesitant now because there giving a null stat file as param would result in an NPE when calling the Optional.of(statisticsFile).
Not sure now if noop is the way to go here or the Precondition was better. Sorry to this back and forth. What do you think?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1969601484
##########
core/src/main/java/org/apache/iceberg/SetPartitionStatistics.java:
##########
@@ -36,7 +35,10 @@ public SetPartitionStatistics(TableOperations ops) {
@Override
public UpdatePartitionStatistics setPartitionStatistics(PartitionStatisticsFile file) {
- Preconditions.checkArgument(null != file, "partition statistics file must not be null");
+ if (file == null) {
Review Comment:
I checked Trino & Spark Puffin implementation, caller is making sure it is not null.
```
IcebergMetadata#finishStatisticsCollection() in Trino
ComputeTableStatsSparkAction#execute() in spark
```
API level NPE is not a good idea. I am ok with no-op as it avoids extra null handling for the callers.
So, I don't think we need to revert back again.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "gaborkaszab (via GitHub)" <gi...@apache.org>.
gaborkaszab commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1969140073
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a combined partition type which considers
+ * all specs in a table.
+ *
+ * @param unifiedPartitionType unified partition schema type. Could be calculated by {@link
+ * Partitioning#partitionType(Table)}.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType unifiedPartitionType) {
+ Preconditions.checkState(!unifiedPartitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), unifiedPartitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, SnapshotRef.MAIN_BRANCH);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null || branch.equals(SnapshotRef.MAIN_BRANCH),
+ "Couldn't find the snapshot for the branch %s",
+ branch);
+ return null;
+ }
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterable<PartitionStats> records)
+ throws IOException {
+ FileFormat fileFormat =
+ FileFormat.fromString(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+
+ OutputFile outputFile = newPartitionStatsFile(table, fileFormat, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile, fileFormat)) {
+ records.iterator().forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static OutputFile newPartitionStatsFile(
+ Table table, FileFormat fileFormat, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(
+ Locale.ROOT, "partition-stats-%d-%s", snapshotId, UUID.randomUUID()))));
Review Comment:
I agree, we shouldn't block this PR with the orphaned stat files issue as it is an existing one already. I just wanted to raise awareness that we should continue that discussion on dev@ to to figure out a solution if there is any.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1818391768
##########
core/src/main/java/org/apache/iceberg/data/PartitionStatsRecord.java:
##########
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.relocated.com.google.common.base.Objects;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.types.Types.StructType;
+
+/** Wraps the {@link PartitionStats} as {@link Record}. Used by generic writers and readers. */
+public class PartitionStatsRecord implements Record, StructLike {
+ private static final LoadingCache<StructType, Map<String, Integer>> NAME_MAP_CACHE =
Review Comment:
> I'm a little confused why we need a special class for this? GenericRecord should work right?
I got a review comment perviously from Anton that keeping the `Record` in the public interface of writer and readers is fragile. So, New class introduced which is less fragile (coupled with partition stats schema and just wraps the PartitionStats).
https://github.com/apache/iceberg/pull/10176#issuecomment-2307890941
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1907057882
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.BiFunction;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.data.avro.DataReader;
+import org.apache.iceberg.data.orc.GenericOrcReader;
+import org.apache.iceberg.data.parquet.GenericParquetReaders;
+import org.apache.iceberg.encryption.EncryptedFiles;
+import org.apache.iceberg.encryption.EncryptionKeyMetadata;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.FileWriterFactory;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.orc.ORC;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
Review Comment:
Wouldn't it be better not to have fat methods with multiple responsibilities? What if we introduce the write method that gets the stats iterator as an argument.
We might not need to execute the complete stats rebuild for all the registered partitions, but only for those that were changed in the current snapshot.
Snapshot summary already has a metric for the number of changed partitions, maybe we could extend it with the partition list and re-compute stats only for them. Then generate a new stats file based on the prev snapshot stats with updates to the changed partitions.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2579821717
@deniskuzZ: Thanks for testing out. We are working on Internal parquet/Avro/orc readers and writes. partition stats will use them. So, we don't need to go through these converters. I will retest all the data types once I use internal writers for partition stats.
PRs:
#11919
#11904
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2593651091
@aokolnychyi, @rdblue, @RussellSpitzer: I have reworked on the PR to use Internal writers and readers. PR is much simpler and no need to handle those conversions now. I can rebase it once the Parquet internal writer PR is merged.
@deniskuzZ : Feel free to test the latest state. It doesn't have conversion layer. So, should behave as expected now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1941184059
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
Review Comment:
Good point.
It is not something which we enforce in Iceberg, but use it some cases.
I'm fine to keep it, but it might be worth to have a clear guideline on this if the community agrees
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1941204512
##########
core/src/test/java/org/apache/iceberg/TestTables.java:
##########
@@ -93,6 +93,26 @@ public static TestTable create(
return new TestTable(ops, name, reporter);
}
+ public static TestTable create(
+ File temp,
+ String name,
+ Schema schema,
+ PartitionSpec spec,
+ int formatVersion,
+ Map<String, String> properties) {
Review Comment:
I think I don't really understand your answer here 😢
I'm not suggesting just simply adding the parameter.
What I'm suggesting is to create a new constructor with the extra parameter, but in the old constructor we should remove the implementation and call the new constructor with an empty property map. This way we will not have duplicated code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1941219705
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
Review Comment:
Should we just keep the generated schema value in a local variable?
I am still not sold on having such an important "Note" in javadoc for a public API. Seems too error prone to me.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1961772494
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records)
+ throws IOException {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile)) {
+ records.forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(Schema dataSchema, OutputFile outputFile)
+ throws IOException {
+ FileFormat fileFormat = fileFormat(outputFile.location());
+ switch (fileFormat) {
+ case PARQUET:
+ return Parquet.writeData(outputFile)
+ .schema(dataSchema)
+ .createWriterFunc(InternalWriter::create)
+ .overwrite()
Review Comment:
Done. Added UUID in the file name.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "gaborkaszab (via GitHub)" <gi...@apache.org>.
gaborkaszab commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1967872485
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
Review Comment:
I'm fine with returning null here. One more thing that comes to my mind: now `UpdatePartitionStatistics.setPartitionStatistics(PartitionStatsFile)` throws because of a Precondition failure if the input file is null. An extra null check is required then when we call that set function.
I'm wondering if it would make sense to simply return in that function if the file is null now that we agreed here that `computeAndWriteStatsFile` is expected to return null if there are no stats. It would make it cleaner to use the code for partition stats
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1939070271
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
Review Comment:
Why not `schema(Table)`? In this case we would not need the "Note" and make sure that it is calculated correctly.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "pvary (via GitHub)" <gi...@apache.org>.
pvary commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1939006203
##########
core/src/main/java/org/apache/iceberg/PartitionStatsUtil.java:
##########
@@ -133,4 +134,27 @@ private static Collection<PartitionStats> mergeStats(
return statsMap.values();
}
+
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ public static boolean isEqual(
+ Comparator<StructLike> partitionComparator, PartitionStats stats1, PartitionStats stats2) {
Review Comment:
Do I understand correctly that this will never been used in production code, just in tests?
Do we publish a test artifact? If so, we can put this code to the test artifact and users can depend on it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "gaborkaszab (via GitHub)" <gi...@apache.org>.
gaborkaszab commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1946350566
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
Review Comment:
Same question as above I guess: null or exception or empty stats
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "jbonofre (via GitHub)" <gi...@apache.org>.
jbonofre commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1970109787
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a combined partition type which considers
+ * all specs in a table.
+ *
+ * @param unifiedPartitionType unified partition schema type. Could be calculated by {@link
+ * Partitioning#partitionType(Table)}.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType unifiedPartitionType) {
+ Preconditions.checkState(!unifiedPartitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), unifiedPartitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, SnapshotRef.MAIN_BRANCH);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null || branch.equals(SnapshotRef.MAIN_BRANCH),
+ "Couldn't find the snapshot for the branch %s",
+ branch);
+ return null;
+ }
+
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats);
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterable<PartitionStats> records)
+ throws IOException {
+ FileFormat fileFormat =
+ FileFormat.fromString(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+
+ OutputFile outputFile = newPartitionStatsFile(table, fileFormat, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile, fileFormat)) {
+ records.iterator().forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static OutputFile newPartitionStatsFile(
+ Table table, FileFormat fileFormat, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(
+ Locale.ROOT, "partition-stats-%d-%s", snapshotId, UUID.randomUUID()))));
Review Comment:
Fair enough. Let's move forward and discuss on dev@
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1961772043
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "Table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) throws IOException {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch, or null if no statistics are
+ * present.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch)
+ throws IOException {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ if (stats.isEmpty()) {
+ return null;
+ }
+
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records)
+ throws IOException {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile)) {
+ records.forEachRemaining(writer::write);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ Preconditions.checkArgument(
+ table instanceof HasTableOperations,
+ "Table must have operations to retrieve metadata location");
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
+ .operations()
+ .metadataFileLocation(
+ fileFormat.addExtension(
+ String.format(Locale.ROOT, "partition-stats-%d", snapshotId))));
+ }
+
+ private static DataWriter<StructLike> dataWriter(Schema dataSchema, OutputFile outputFile)
+ throws IOException {
+ FileFormat fileFormat = fileFormat(outputFile.location());
Review Comment:
done. Passed down as the argument
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2670469872
Rebased and PR is ready.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1961697310
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
+ return table
+ .io()
+ .newOutputFile(
+ ((HasTableOperations) table)
Review Comment:
because above I have added a precondition to check `HasTableOperations`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1940558710
##########
core/src/test/java/org/apache/iceberg/TestTables.java:
##########
@@ -93,6 +93,26 @@ public static TestTable create(
return new TestTable(ops, name, reporter);
}
+ public static TestTable create(
+ File temp,
+ String name,
+ Schema schema,
+ PartitionSpec spec,
+ int formatVersion,
+ Map<String, String> properties) {
Review Comment:
Followed a similar style when they added `MetricsReporter`,
> Why not adding the parameter to the old create
It is public. So, need to modify all the callers.
But we can refactor a private method that can be helper to all these public method. I can do it in a follow up to keep minimal changes for this PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1940535942
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
+
+ private static OutputFile newPartitionStatsFile(Table table, long snapshotId) {
+ FileFormat fileFormat =
+ fileFormat(
+ table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
Review Comment:
For the reader code, it has to infer from the input file extension. To keep reader and writer signature similar. It has done like this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1940531115
##########
data/src/main/java/org/apache/iceberg/data/PartitionStatsHandler.java:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.PartitionStatsUtil;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.InternalReader;
+import org.apache.iceberg.data.parquet.InternalWriter;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.LongType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.StructType;
+import org.apache.iceberg.util.SnapshotUtil;
+
+/**
+ * Computes, writes and reads the {@link PartitionStatisticsFile}. Uses generic readers and writers
+ * to support writing and reading of the stats in table default format.
+ */
+public final class PartitionStatsHandler {
+
+ private PartitionStatsHandler() {}
+
+ public enum Column {
+ PARTITION(0),
+ SPEC_ID(1),
+ DATA_RECORD_COUNT(2),
+ DATA_FILE_COUNT(3),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES(4),
+ POSITION_DELETE_RECORD_COUNT(5),
+ POSITION_DELETE_FILE_COUNT(6),
+ EQUALITY_DELETE_RECORD_COUNT(7),
+ EQUALITY_DELETE_FILE_COUNT(8),
+ TOTAL_RECORD_COUNT(9),
+ LAST_UPDATED_AT(10),
+ LAST_UPDATED_SNAPSHOT_ID(11);
+
+ private final int id;
+
+ Column(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+ }
+
+ /**
+ * Generates the partition stats file schema based on a given partition type.
+ *
+ * <p>Note: Provide the unified partition schema type as mentioned in the spec.
+ *
+ * @param partitionType unified partition schema type.
+ * @return a schema that corresponds to the provided unified partition type.
+ */
+ public static Schema schema(StructType partitionType) {
+ Preconditions.checkState(!partitionType.fields().isEmpty(), "table must be partitioned");
+ return new Schema(
+ NestedField.required(1, Column.PARTITION.name(), partitionType),
+ NestedField.required(2, Column.SPEC_ID.name(), IntegerType.get()),
+ NestedField.required(3, Column.DATA_RECORD_COUNT.name(), LongType.get()),
+ NestedField.required(4, Column.DATA_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.required(5, Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.name(), LongType.get()),
+ NestedField.optional(6, Column.POSITION_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(7, Column.POSITION_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(8, Column.EQUALITY_DELETE_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(9, Column.EQUALITY_DELETE_FILE_COUNT.name(), IntegerType.get()),
+ NestedField.optional(10, Column.TOTAL_RECORD_COUNT.name(), LongType.get()),
+ NestedField.optional(11, Column.LAST_UPDATED_AT.name(), LongType.get()),
+ NestedField.optional(12, Column.LAST_UPDATED_SNAPSHOT_ID.name(), LongType.get()));
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table's current snapshot.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @return {@link PartitionStatisticsFile} for the current snapshot.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table) {
+ return computeAndWriteStatsFile(table, null);
+ }
+
+ /**
+ * Computes and writes the {@link PartitionStatisticsFile} for a given table and branch.
+ *
+ * @param table The {@link Table} for which the partition statistics is computed.
+ * @param branch A branch information to select the required snapshot.
+ * @return {@link PartitionStatisticsFile} for the given branch.
+ */
+ public static PartitionStatisticsFile computeAndWriteStatsFile(Table table, String branch) {
+ Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
+ if (currentSnapshot == null) {
+ Preconditions.checkArgument(
+ branch == null, "Couldn't find the snapshot for the branch %s", branch);
+ return null;
+ }
+
+ StructType partitionType = Partitioning.partitionType(table);
+ Collection<PartitionStats> stats = PartitionStatsUtil.computeStats(table, currentSnapshot);
+ List<PartitionStats> sortedStats = PartitionStatsUtil.sortStats(stats, partitionType);
+ return writePartitionStatsFile(
+ table, currentSnapshot.snapshotId(), schema(partitionType), sortedStats.iterator());
+ }
+
+ @VisibleForTesting
+ static PartitionStatisticsFile writePartitionStatsFile(
+ Table table, long snapshotId, Schema dataSchema, Iterator<PartitionStats> records) {
+ OutputFile outputFile = newPartitionStatsFile(table, snapshotId);
+
+ try (DataWriter<StructLike> writer = dataWriter(dataSchema, outputFile); ) {
+ records.forEachRemaining(writer::write);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .path(outputFile.location())
+ .fileSizeInBytes(outputFile.toInputFile().getLength())
+ .build();
+ }
+
+ /**
+ * Reads partition statistics from the specified {@link InputFile} using given schema.
+ *
+ * @param schema The {@link Schema} of the partition statistics file.
+ * @param inputFile An {@link InputFile} pointing to the partition stats file.
+ */
+ public static CloseableIterable<PartitionStats> readPartitionStatsFile(
+ Schema schema, InputFile inputFile) {
+ CloseableIterable<StructLike> records = dataReader(schema, inputFile);
+ return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats);
+ }
+
+ private static FileFormat fileFormat(String fileLocation) {
+ return FileFormat.fromString(fileLocation.substring(fileLocation.lastIndexOf(".") + 1));
+ }
Review Comment:
This is private method and it is based on `newPartitionStatsFile`. It was a way to avoid passing more arguments.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1777914781
##########
core/src/main/java/org/apache/iceberg/data/PartitionStatsRecord.java:
##########
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.relocated.com.google.common.base.Objects;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.types.Types.StructType;
+
+/** Wraps the {@link PartitionStats} as {@link Record}. Used by generic writers and readers. */
+public class PartitionStatsRecord implements Record, StructLike {
+ private static final LoadingCache<StructType, Map<String, Integer>> NAME_MAP_CACHE =
Review Comment:
Similar to `GenericRecord` but for a specific partition stats schema.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2434651434
@aokolnychyi, @rdblue:
I already tried POC for internal writers on another branch,
https://github.com/apache/iceberg/commit/c209bc94c3a43e3fb8262cffa121dc450889dfd8
The problems:
a) I am using PartitionData instead of Record for partition value, but the `PartitionData` get() method wraps the byte array which is a problem for internal writers, they expect byte[]. So, I didn't felt like using a new class instead of `PartitionData` just for this.
b) Also, Using partitionData in StructLikeMap is not working fine. Some keys are missing in the map (looks like equals() logic), If I use Record, it is fine.
Maybe in the next version we can have optimized writer and reader (without converter using internal reader and writers).
For end user it doesn't make any difference as new readers can also read the old partition stats parquet file and old readers can read the new partition stats parquet file. So, can we merge this?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "RussellSpitzer (via GitHub)" <gi...@apache.org>.
RussellSpitzer commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1816945414
##########
core/src/test/java/org/apache/iceberg/TestTables.java:
##########
@@ -93,6 +93,26 @@ public static TestTable create(
return new TestTable(ops, name, reporter);
}
+ public static TestTable create(
+ File temp,
+ String name,
+ Schema schema,
+ PartitionSpec spec,
+ int formatVersion,
+ Map<String, String> properties) {
+ TestTableOperations ops = new TestTableOperations(name, temp);
+ if (ops.current() != null) {
+ throw new AlreadyExistsException("Table %s already exists at location: %s", name, temp);
Review Comment:
Why this check? I think it makes sense but wondering why it's being added now
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "aokolnychyi (via GitHub)" <gi...@apache.org>.
aokolnychyi commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2434273228
I think we should try to use "internal" writers. @rdblue added "internal" readers recently.
Any guidance on how to add a writer, @rdblue? We can start with Avro for now. We will also need such readers/writers for Parquet.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1818418685
##########
core/src/main/java/org/apache/iceberg/PartitionStats.java:
##########
@@ -249,4 +250,45 @@ public <T> void set(int pos, T value) {
throw new UnsupportedOperationException("Unknown position: " + pos);
}
}
+
+ @Override
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ } else if (!(other instanceof PartitionStats)) {
+ return false;
+ }
+
+ PartitionStats that = (PartitionStats) other;
+ return Objects.equals(partition, that.partition)
Review Comment:
We are storing `Record` as `Partition` from `coercedPartitionRecord()`.
Since `GenericRecord` has `equals` implemented, calling `Objects.equals` is working here.
Hence, I didn't add comparator logic.
But agree that need to understand the implementation logic, just by looking at this class, it looks like we need comparator logic. I can update it if it is necessary.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "RussellSpitzer (via GitHub)" <gi...@apache.org>.
RussellSpitzer commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1819348590
##########
core/src/main/java/org/apache/iceberg/data/PartitionStatsRecord.java:
##########
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.relocated.com.google.common.base.Objects;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.types.Types.StructType;
+
+/** Wraps the {@link PartitionStats} as {@link Record}. Used by generic writers and readers. */
+public class PartitionStatsRecord implements Record, StructLike {
+ private static final LoadingCache<StructType, Map<String, Integer>> NAME_MAP_CACHE =
Review Comment:
I'm not sure why we need a special class here still? His comment is just to remove Record from the public interface which has been done. I don't think creating a new special class (which is public) is necessary since the records only exist within private handler code? - @aokolnychyi was in Europe last I checked but when he is back he can check it out.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "RussellSpitzer (via GitHub)" <gi...@apache.org>.
RussellSpitzer commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2442317326
Moving out of 1.7.0 since we still have a bit of discussion here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "RussellSpitzer (via GitHub)" <gi...@apache.org>.
RussellSpitzer commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1816930724
##########
core/src/main/java/org/apache/iceberg/PartitionStats.java:
##########
@@ -249,4 +250,45 @@ public <T> void set(int pos, T value) {
throw new UnsupportedOperationException("Unknown position: " + pos);
}
}
+
+ @Override
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ } else if (!(other instanceof PartitionStats)) {
+ return false;
+ }
+
+ PartitionStats that = (PartitionStats) other;
+ return Objects.equals(partition, that.partition)
Review Comment:
StructLike doesn't have equal, I think you need to use StructLikeComparator here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on code in PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#discussion_r1822062905
##########
core/src/main/java/org/apache/iceberg/data/PartitionStatsRecord.java:
##########
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.relocated.com.google.common.base.Objects;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.types.Types.StructType;
+
+/** Wraps the {@link PartitionStats} as {@link Record}. Used by generic writers and readers. */
+public class PartitionStatsRecord implements Record, StructLike {
+ private static final LoadingCache<StructType, Map<String, Integer>> NAME_MAP_CACHE =
Review Comment:
Also, `GenericRecord` can't wrap the `partitionStats`, it maintains its own data array.
##########
core/src/main/java/org/apache/iceberg/data/PartitionStatsRecord.java:
##########
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.PartitionStats;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.relocated.com.google.common.base.Objects;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.types.Types.StructType;
+
+/** Wraps the {@link PartitionStats} as {@link Record}. Used by generic writers and readers. */
+public class PartitionStatsRecord implements Record, StructLike {
+ private static final LoadingCache<StructType, Map<String, Integer>> NAME_MAP_CACHE =
Review Comment:
Also, `GenericRecord` can't wrap the `PartitionStats`, it maintains its own data array.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "ajantha-bhat (via GitHub)" <gi...@apache.org>.
ajantha-bhat commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2486040761
@RussellSpitzer: I have added the Assert and replied to https://github.com/apache/iceberg/pull/11216#discussion_r1822062905, do you have anymore comments for this PR?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2579758121
hi @ajantha-bhat, could you please check below:
it seems that Date, time, timestamp partition values are not properly serialized
see Type.TypeID
````
DATE(Integer.class),
TIME(Long.class),
TIMESTAMP(Long.class),
````
PartitionSpec.partitionToPath(PartitionStatsRecord.unwrap().partition()) thows an exception
````
Caused by: java.lang.IllegalStateException: Not an instance of java.lang.Integer: 1999-12-12
at org.apache.iceberg.data.GenericRecord.get(GenericRecord.java:123)
at org.apache.iceberg.PartitionSpec.get(PartitionSpec.java:195)
at org.apache.iceberg.PartitionSpec.partitionToPath(PartitionSpec.java:213)
````
I think, instead of Record(1999-12-12) it should be Record{10956}
full code snippet
````
Types.StructType partitionType = Partitioning.partitionType(table);
Schema schema = PartitionStatsHandler.schema(partitionType);
CloseableIterable<PartitionStatsRecord> partitionStatsRecords = PartitionStatsHandler.readPartitionStatsFile(
schema, table.io().newInputFile(statsFile.path()));
try (Closeable toClose = partitionStatsRecords) {
PartitionStats partitionStats = Iterables.tryFind(partitionStatsRecords, stats -> {
PartitionSpec spec = table.specs().get(stats.unwrap().specId());
return spec.partitionToPath(stats.unwrap().partition()).equals(partish.getPartition().getName());
})
.transform(PartitionStatsRecord::unwrap)
.orNull();
if (partitionStats != null) {
Map<String, String> stats = ImmutableMap.of(
TOTAL_DATA_FILES_PROP, String.valueOf(partitionStats.dataFileCount()),
TOTAL_RECORDS_PROP, String.valueOf(partitionStats.dataRecordCount()),
TOTAL_EQ_DELETES_PROP, String.valueOf(partitionStats.equalityDeleteRecordCount()),
TOTAL_POS_DELETES_PROP, String.valueOf(partitionStats.positionDeleteRecordCount()),
TOTAL_FILE_SIZE_PROP, String.valueOf(partitionStats.totalDataFileSizeInBytes())
);
return stats;
````
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] Data: Add partition stats writer and reader [iceberg]
Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on PR #11216:
URL: https://github.com/apache/iceberg/pull/11216#issuecomment-2512537342
Thanks @ajantha-bhat for your work on partition stats support in Iceberg! That could be reused in Hive as a building block for https://github.com/apache/hive/pull/5498
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org