You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@paimon.apache.org by "0dunay0 (via GitHub)" <gi...@apache.org> on 2026/04/26 23:27:19 UTC
Re: [PR] [core/iceberg] Added optional snapshot summary fields to iceberg metadata [paimon]
0dunay0 commented on code in PR #6503:
URL: https://github.com/apache/paimon/pull/6503#discussion_r3144344205
##########
paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java:
##########
@@ -1230,6 +1328,147 @@ private List<IcebergManifestFileMeta> createDvManifestFileMetas(Snapshot snapsho
icebergDvEntries.iterator(), snapshotId, IcebergManifestFileMeta.Content.DELETES);
}
+ // -------------------------------------------------------------------------------------
+ // Snapshot Summary Computation
+ // -------------------------------------------------------------------------------------
+
+ private static class SummaryMetrics {
+ long addedDataFiles;
+ long addedRecords;
+ long addedFilesSize;
+ long deletedDataFiles;
+ long deletedRecords;
+ long deletedFilesSize;
+ long changedPartitionCount;
+ long totalDataFiles;
+ long totalRecords;
+ long totalFilesSize;
+ long totalDeleteFiles;
+ long totalPositionDeletes;
+ long totalEqualityDeletes;
+ }
+
+ private IcebergSnapshotSummary computeSnapshotSummary(
+ String operation, Snapshot snapshot, SummaryMetrics metrics) {
+
+ IcebergSnapshotSummary summary = new IcebergSnapshotSummary(operation);
+
+ long addedDataFiles = Math.max(0, metrics.addedDataFiles);
+ long addedRecords = Math.max(0, metrics.addedRecords);
+ long addedFilesSize = Math.max(0, metrics.addedFilesSize);
+ long deletedDataFiles = Math.max(0, metrics.deletedDataFiles);
+ long deletedRecords = Math.max(0, metrics.deletedRecords);
+ long deletedFilesSize = Math.max(0, metrics.deletedFilesSize);
+ long changedPartitionCount = Math.max(0, metrics.changedPartitionCount);
+ long totalRecords = Math.max(0, metrics.totalRecords);
+ long totalDataFiles = Math.max(0, metrics.totalDataFiles);
+ long totalFilesSize = Math.max(0, metrics.totalFilesSize);
+ long totalDeleteFiles = Math.max(0, metrics.totalDeleteFiles);
+ long totalPositionDeletes = Math.max(0, metrics.totalPositionDeletes);
+ long totalEqualityDeletes = Math.max(0, metrics.totalEqualityDeletes);
+
+ summary.put("added-data-files", Long.toString(addedDataFiles));
Review Comment:
Added the metrics as constants.
##########
paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java:
##########
@@ -1230,6 +1328,147 @@ private List<IcebergManifestFileMeta> createDvManifestFileMetas(Snapshot snapsho
icebergDvEntries.iterator(), snapshotId, IcebergManifestFileMeta.Content.DELETES);
}
+ // -------------------------------------------------------------------------------------
+ // Snapshot Summary Computation
+ // -------------------------------------------------------------------------------------
+
+ private static class SummaryMetrics {
+ long addedDataFiles;
+ long addedRecords;
+ long addedFilesSize;
+ long deletedDataFiles;
+ long deletedRecords;
+ long deletedFilesSize;
+ long changedPartitionCount;
+ long totalDataFiles;
+ long totalRecords;
+ long totalFilesSize;
+ long totalDeleteFiles;
+ long totalPositionDeletes;
+ long totalEqualityDeletes;
+ }
+
+ private IcebergSnapshotSummary computeSnapshotSummary(
+ String operation, Snapshot snapshot, SummaryMetrics metrics) {
+
+ IcebergSnapshotSummary summary = new IcebergSnapshotSummary(operation);
+
+ long addedDataFiles = Math.max(0, metrics.addedDataFiles);
+ long addedRecords = Math.max(0, metrics.addedRecords);
+ long addedFilesSize = Math.max(0, metrics.addedFilesSize);
+ long deletedDataFiles = Math.max(0, metrics.deletedDataFiles);
+ long deletedRecords = Math.max(0, metrics.deletedRecords);
+ long deletedFilesSize = Math.max(0, metrics.deletedFilesSize);
+ long changedPartitionCount = Math.max(0, metrics.changedPartitionCount);
+ long totalRecords = Math.max(0, metrics.totalRecords);
+ long totalDataFiles = Math.max(0, metrics.totalDataFiles);
+ long totalFilesSize = Math.max(0, metrics.totalFilesSize);
+ long totalDeleteFiles = Math.max(0, metrics.totalDeleteFiles);
+ long totalPositionDeletes = Math.max(0, metrics.totalPositionDeletes);
+ long totalEqualityDeletes = Math.max(0, metrics.totalEqualityDeletes);
+
+ summary.put("added-data-files", Long.toString(addedDataFiles));
+ summary.put("added-records", Long.toString(addedRecords));
+ summary.put("added-files-size", Long.toString(addedFilesSize));
+ summary.put("deleted-data-files", Long.toString(deletedDataFiles));
+ summary.put("deleted-records", Long.toString(deletedRecords));
+ summary.put("deleted-files-size", Long.toString(deletedFilesSize));
+ summary.put("changed-partition-count", Long.toString(changedPartitionCount));
+ summary.put("total-records", Long.toString(totalRecords));
+ summary.put("total-data-files", Long.toString(totalDataFiles));
+ summary.put("total-files-size", Long.toString(totalFilesSize));
+ summary.put("total-delete-files", Long.toString(totalDeleteFiles));
+ summary.put("total-position-deletes", Long.toString(totalPositionDeletes));
+ summary.put("total-equality-deletes", Long.toString(totalEqualityDeletes));
+
+ Map<String, String> properties = snapshot.properties();
+ if (properties != null) {
+ properties.forEach(
+ (key, value) -> {
+ if (value != null) {
+ summary.put(key, value);
+ }
+ });
+ }
+
+ return summary;
+ }
+
+ private long computeLiveDataFileCount(List<IcebergManifestFileMeta> manifestMetas) {
+ return manifestMetas.stream()
+ .mapToLong(
+ meta ->
+ meta.addedFilesCount()
+ + meta.existingFilesCount()
+ - meta.deletedFilesCount())
+ .sum();
+ }
+
+ private long computeLiveRowCount(List<IcebergManifestFileMeta> manifestMetas) {
+ return manifestMetas.stream()
+ .mapToLong(
+ meta ->
+ meta.addedRowsCount()
+ + meta.existingRowsCount()
+ - meta.deletedRowsCount())
+ .sum();
+ }
+
+ private long computeLiveDeleteFileCount(List<IcebergManifestFileMeta> manifestMetas) {
+ return manifestMetas.stream()
+ .mapToLong(
+ meta ->
+ meta.addedFilesCount()
+ + meta.existingFilesCount()
+ - meta.deletedFilesCount())
+ .sum();
+ }
+
+ private long computeDeleteRowCount(List<IcebergManifestFileMeta> manifestMetas) {
Review Comment:
I removed the duplicate and renamed the shared method to `computeLiveFileCount`. I also found that `total-records` was using `paimonSnapshot.totalRecordCount()` which includes level-0 files that Iceberg can't see in PK tables. Switched it to the same incremental pattern used by `total-data-files` and `total-files-size`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org