You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by fo...@apache.org on 2022/09/21 08:56:27 UTC
[iceberg] branch master updated: API,Core: Add scan planning metrics for skipped data/delete files (#5788)
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 15b3e2f300 API,Core: Add scan planning metrics for skipped data/delete files (#5788)
15b3e2f300 is described below
commit 15b3e2f3007d7a2e1ce2b35351db2b43a19e986d
Author: Eduard Tudenhöfner <et...@gmail.com>
AuthorDate: Wed Sep 21 10:56:20 2022 +0200
API,Core: Add scan planning metrics for skipped data/delete files (#5788)
---
.../org/apache/iceberg/metrics/DefaultCounter.java | 5 +
.../org/apache/iceberg/metrics/ScanReport.java | 20 ++++
.../java/org/apache/iceberg/metrics/Timer.java | 5 +
.../java/org/apache/iceberg/DeleteFileIndex.java | 1 +
.../java/org/apache/iceberg/ManifestGroup.java | 12 +-
.../java/org/apache/iceberg/ManifestReader.java | 9 +-
.../iceberg/metrics/ScanMetricsResultParser.java | 12 ++
.../iceberg/TestScanPlanningAndReporting.java | 127 ++++++++++++++++-----
.../metrics/TestScanMetricsResultParser.java | 45 ++++++++
.../iceberg/metrics/TestScanReportParser.java | 14 +++
10 files changed, 216 insertions(+), 34 deletions(-)
diff --git a/api/src/main/java/org/apache/iceberg/metrics/DefaultCounter.java b/api/src/main/java/org/apache/iceberg/metrics/DefaultCounter.java
index 8968a4377d..9bb23092a8 100644
--- a/api/src/main/java/org/apache/iceberg/metrics/DefaultCounter.java
+++ b/api/src/main/java/org/apache/iceberg/metrics/DefaultCounter.java
@@ -38,6 +38,11 @@ public class DefaultCounter implements Counter {
public long value() {
throw new UnsupportedOperationException("NOOP counter has no value");
}
+
+ @Override
+ public String toString() {
+ return "NOOP counter";
+ }
};
private final LongAdder counter;
diff --git a/api/src/main/java/org/apache/iceberg/metrics/ScanReport.java b/api/src/main/java/org/apache/iceberg/metrics/ScanReport.java
index e8d4918c4f..304821199d 100644
--- a/api/src/main/java/org/apache/iceberg/metrics/ScanReport.java
+++ b/api/src/main/java/org/apache/iceberg/metrics/ScanReport.java
@@ -125,6 +125,12 @@ public interface ScanReport {
@Nullable
CounterResult totalDeleteFileSizeInBytes();
+ @Nullable
+ CounterResult skippedDataFiles();
+
+ @Nullable
+ CounterResult skippedDeleteFiles();
+
static ScanMetricsResult fromScanMetrics(ScanMetrics scanMetrics) {
Preconditions.checkArgument(null != scanMetrics, "Invalid scan metrics: null");
return ImmutableScanMetricsResult.builder()
@@ -138,6 +144,8 @@ public interface ScanReport {
.totalFileSizeInBytes(CounterResult.fromCounter(scanMetrics.totalFileSizeInBytes()))
.totalDeleteFileSizeInBytes(
CounterResult.fromCounter(scanMetrics.totalDeleteFileSizeInBytes()))
+ .skippedDataFiles(CounterResult.fromCounter(scanMetrics.skippedDataFiles()))
+ .skippedDeleteFiles(CounterResult.fromCounter(scanMetrics.skippedDeleteFiles()))
.build();
}
}
@@ -154,6 +162,8 @@ public interface ScanReport {
public static final String TOTAL_FILE_SIZE_IN_BYTES = "total-file-size-in-bytes";
public static final String TOTAL_DELETE_FILE_SIZE_IN_BYTES = "total-delete-file-size-in-bytes";
public static final String SKIPPED_DATA_MANIFESTS = "skipped-data-manifests";
+ public static final String SKIPPED_DATA_FILES = "skipped-data-files";
+ public static final String SKIPPED_DELETE_FILES = "skipped-delete-files";
public static ScanMetrics noop() {
return ScanMetrics.of(MetricsContext.nullMetrics());
@@ -206,6 +216,16 @@ public interface ScanReport {
return metricsContext().counter(SKIPPED_DATA_MANIFESTS, MetricsContext.Unit.COUNT);
}
+ @Value.Derived
+ public Counter skippedDataFiles() {
+ return metricsContext().counter(SKIPPED_DATA_FILES, MetricsContext.Unit.COUNT);
+ }
+
+ @Value.Derived
+ public Counter skippedDeleteFiles() {
+ return metricsContext().counter(SKIPPED_DELETE_FILES, MetricsContext.Unit.COUNT);
+ }
+
public static ScanMetrics of(MetricsContext metricsContext) {
return ImmutableScanMetrics.builder().metricsContext(metricsContext).build();
}
diff --git a/api/src/main/java/org/apache/iceberg/metrics/Timer.java b/api/src/main/java/org/apache/iceberg/metrics/Timer.java
index 03ecc5fbba..abef48845b 100644
--- a/api/src/main/java/org/apache/iceberg/metrics/Timer.java
+++ b/api/src/main/java/org/apache/iceberg/metrics/Timer.java
@@ -165,5 +165,10 @@ public interface Timer {
public <T> T time(Supplier<T> supplier) {
return supplier.get();
}
+
+ @Override
+ public String toString() {
+ return "NOOP timer";
+ }
};
}
diff --git a/core/src/main/java/org/apache/iceberg/DeleteFileIndex.java b/core/src/main/java/org/apache/iceberg/DeleteFileIndex.java
index 4a8425328a..0746ea26d0 100644
--- a/core/src/main/java/org/apache/iceberg/DeleteFileIndex.java
+++ b/core/src/main/java/org/apache/iceberg/DeleteFileIndex.java
@@ -547,6 +547,7 @@ class DeleteFileIndex {
.filterPartitions(partitionFilter)
.filterPartitions(partitionSet)
.caseSensitive(caseSensitive)
+ .scanMetrics(scanMetrics)
.liveEntries());
}
}
diff --git a/core/src/main/java/org/apache/iceberg/ManifestGroup.java b/core/src/main/java/org/apache/iceberg/ManifestGroup.java
index 32bcf1d153..af02b0ebc3 100644
--- a/core/src/main/java/org/apache/iceberg/ManifestGroup.java
+++ b/core/src/main/java/org/apache/iceberg/ManifestGroup.java
@@ -311,16 +311,22 @@ class ManifestGroup {
if (ignoreExisting) {
entries =
CloseableIterable.filter(
- entries, entry -> entry.status() != ManifestEntry.Status.EXISTING);
+ scanMetrics.skippedDataFiles(),
+ entries,
+ entry -> entry.status() != ManifestEntry.Status.EXISTING);
}
if (evaluator != null) {
entries =
CloseableIterable.filter(
- entries, entry -> evaluator.eval((GenericDataFile) entry.file()));
+ scanMetrics.skippedDataFiles(),
+ entries,
+ entry -> evaluator.eval((GenericDataFile) entry.file()));
}
- entries = CloseableIterable.filter(entries, manifestEntryPredicate);
+ entries =
+ CloseableIterable.filter(
+ scanMetrics.skippedDataFiles(), entries, manifestEntryPredicate);
iterable = entryFn.apply(manifest, entries);
diff --git a/core/src/main/java/org/apache/iceberg/ManifestReader.java b/core/src/main/java/org/apache/iceberg/ManifestReader.java
index e60fa4a477..63129ecdb0 100644
--- a/core/src/main/java/org/apache/iceberg/ManifestReader.java
+++ b/core/src/main/java/org/apache/iceberg/ManifestReader.java
@@ -206,6 +206,9 @@ public class ManifestReader<F extends ContentFile<F>> extends CloseableGroup
requireStatsProjection ? withStatsColumns(columns) : columns;
return CloseableIterable.filter(
+ content == FileType.DATA_FILES
+ ? scanMetrics.skippedDataFiles()
+ : scanMetrics.skippedDeleteFiles(),
open(projection(fileSchema, fileProjection, projectColumns, caseSensitive)),
entry ->
entry != null
@@ -255,7 +258,11 @@ public class ManifestReader<F extends ContentFile<F>> extends CloseableGroup
CloseableIterable<ManifestEntry<F>> liveEntries() {
return CloseableIterable.filter(
- entries(), entry -> entry != null && entry.status() != ManifestEntry.Status.DELETED);
+ content == FileType.DATA_FILES
+ ? scanMetrics.skippedDataFiles()
+ : scanMetrics.skippedDeleteFiles(),
+ entries(),
+ entry -> entry != null && entry.status() != ManifestEntry.Status.DELETED);
}
/** @return an Iterator of DataFile. Makes defensive copies of files before returning */
diff --git a/core/src/main/java/org/apache/iceberg/metrics/ScanMetricsResultParser.java b/core/src/main/java/org/apache/iceberg/metrics/ScanMetricsResultParser.java
index 6917450f50..470de5c714 100644
--- a/core/src/main/java/org/apache/iceberg/metrics/ScanMetricsResultParser.java
+++ b/core/src/main/java/org/apache/iceberg/metrics/ScanMetricsResultParser.java
@@ -87,6 +87,16 @@ class ScanMetricsResultParser {
CounterResultParser.toJson(metrics.totalDeleteFileSizeInBytes(), gen);
}
+ if (null != metrics.skippedDataFiles()) {
+ gen.writeFieldName(ScanMetrics.SKIPPED_DATA_FILES);
+ CounterResultParser.toJson(metrics.skippedDataFiles(), gen);
+ }
+
+ if (null != metrics.skippedDeleteFiles()) {
+ gen.writeFieldName(ScanMetrics.SKIPPED_DELETE_FILES);
+ CounterResultParser.toJson(metrics.skippedDeleteFiles(), gen);
+ }
+
gen.writeEndObject();
}
@@ -115,6 +125,8 @@ class ScanMetricsResultParser {
CounterResultParser.fromJson(ScanMetrics.TOTAL_FILE_SIZE_IN_BYTES, json))
.totalDeleteFileSizeInBytes(
CounterResultParser.fromJson(ScanMetrics.TOTAL_DELETE_FILE_SIZE_IN_BYTES, json))
+ .skippedDataFiles(CounterResultParser.fromJson(ScanMetrics.SKIPPED_DATA_FILES, json))
+ .skippedDeleteFiles(CounterResultParser.fromJson(ScanMetrics.SKIPPED_DELETE_FILES, json))
.build();
}
}
diff --git a/core/src/test/java/org/apache/iceberg/TestScanPlanningAndReporting.java b/core/src/test/java/org/apache/iceberg/TestScanPlanningAndReporting.java
index 447cfe9a77..eefe7fc128 100644
--- a/core/src/test/java/org/apache/iceberg/TestScanPlanningAndReporting.java
+++ b/core/src/test/java/org/apache/iceberg/TestScanPlanningAndReporting.java
@@ -27,6 +27,7 @@ import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.metrics.LoggingScanReporter;
import org.apache.iceberg.metrics.ScanReport;
+import org.apache.iceberg.metrics.ScanReport.ScanMetricsResult;
import org.apache.iceberg.metrics.ScanReporter;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.junit.Test;
@@ -61,16 +62,16 @@ public class TestScanPlanningAndReporting extends TableTestBase {
assertThat(scanReport.tableName()).isEqualTo(tableName);
assertThat(scanReport.snapshotId()).isEqualTo(2L);
- assertThat(scanReport.scanMetrics().totalPlanningDuration().totalDuration())
- .isGreaterThan(Duration.ZERO);
- assertThat(scanReport.scanMetrics().resultDataFiles().value()).isEqualTo(3);
- assertThat(scanReport.scanMetrics().resultDeleteFiles().value()).isEqualTo(0);
- assertThat(scanReport.scanMetrics().scannedDataManifests().value()).isEqualTo(2);
- assertThat(scanReport.scanMetrics().skippedDataManifests().value()).isEqualTo(0);
- assertThat(scanReport.scanMetrics().totalDataManifests().value()).isEqualTo(2);
- assertThat(scanReport.scanMetrics().totalDeleteManifests().value()).isEqualTo(0);
- assertThat(scanReport.scanMetrics().totalFileSizeInBytes().value()).isEqualTo(30L);
- assertThat(scanReport.scanMetrics().totalDeleteFileSizeInBytes().value()).isEqualTo(0L);
+ ScanMetricsResult result = scanReport.scanMetrics();
+ assertThat(result.totalPlanningDuration().totalDuration()).isGreaterThan(Duration.ZERO);
+ assertThat(result.resultDataFiles().value()).isEqualTo(3);
+ assertThat(result.resultDeleteFiles().value()).isEqualTo(0);
+ assertThat(result.scannedDataManifests().value()).isEqualTo(2);
+ assertThat(result.skippedDataManifests().value()).isEqualTo(0);
+ assertThat(result.totalDataManifests().value()).isEqualTo(2);
+ assertThat(result.totalDeleteManifests().value()).isEqualTo(0);
+ assertThat(result.totalFileSizeInBytes().value()).isEqualTo(30L);
+ assertThat(result.totalDeleteFileSizeInBytes().value()).isEqualTo(0L);
// we should hit only a single data manifest and only a single data file
try (CloseableIterable<FileScanTask> fileScanTasks =
@@ -79,19 +80,19 @@ public class TestScanPlanningAndReporting extends TableTestBase {
}
scanReport = reporter.lastReport();
+ result = scanReport.scanMetrics();
assertThat(scanReport).isNotNull();
assertThat(scanReport.tableName()).isEqualTo(tableName);
assertThat(scanReport.snapshotId()).isEqualTo(2L);
- assertThat(scanReport.scanMetrics().totalPlanningDuration().totalDuration())
- .isGreaterThan(Duration.ZERO);
- assertThat(scanReport.scanMetrics().resultDataFiles().value()).isEqualTo(1);
- assertThat(scanReport.scanMetrics().resultDeleteFiles().value()).isEqualTo(0);
- assertThat(scanReport.scanMetrics().scannedDataManifests().value()).isEqualTo(1);
- assertThat(scanReport.scanMetrics().skippedDataManifests().value()).isEqualTo(1);
- assertThat(scanReport.scanMetrics().totalDataManifests().value()).isEqualTo(2);
- assertThat(scanReport.scanMetrics().totalDeleteManifests().value()).isEqualTo(0);
- assertThat(scanReport.scanMetrics().totalFileSizeInBytes().value()).isEqualTo(10L);
- assertThat(scanReport.scanMetrics().totalDeleteFileSizeInBytes().value()).isEqualTo(0L);
+ assertThat(result.totalPlanningDuration().totalDuration()).isGreaterThan(Duration.ZERO);
+ assertThat(result.resultDataFiles().value()).isEqualTo(1);
+ assertThat(result.resultDeleteFiles().value()).isEqualTo(0);
+ assertThat(result.scannedDataManifests().value()).isEqualTo(1);
+ assertThat(result.skippedDataManifests().value()).isEqualTo(1);
+ assertThat(result.totalDataManifests().value()).isEqualTo(2);
+ assertThat(result.totalDeleteManifests().value()).isEqualTo(0);
+ assertThat(result.totalFileSizeInBytes().value()).isEqualTo(10L);
+ assertThat(result.totalDeleteFileSizeInBytes().value()).isEqualTo(0L);
}
@Test
@@ -118,16 +119,82 @@ public class TestScanPlanningAndReporting extends TableTestBase {
assertThat(scanReport).isNotNull();
assertThat(scanReport.tableName()).isEqualTo("scan-planning-with-deletes");
assertThat(scanReport.snapshotId()).isEqualTo(2L);
- assertThat(scanReport.scanMetrics().totalPlanningDuration().totalDuration())
- .isGreaterThan(Duration.ZERO);
- assertThat(scanReport.scanMetrics().resultDataFiles().value()).isEqualTo(3);
- assertThat(scanReport.scanMetrics().resultDeleteFiles().value()).isEqualTo(2);
- assertThat(scanReport.scanMetrics().scannedDataManifests().value()).isEqualTo(1);
- assertThat(scanReport.scanMetrics().skippedDataManifests().value()).isEqualTo(0);
- assertThat(scanReport.scanMetrics().totalDataManifests().value()).isEqualTo(1);
- assertThat(scanReport.scanMetrics().totalDeleteManifests().value()).isEqualTo(1);
- assertThat(scanReport.scanMetrics().totalFileSizeInBytes().value()).isEqualTo(30L);
- assertThat(scanReport.scanMetrics().totalDeleteFileSizeInBytes().value()).isEqualTo(20L);
+ ScanMetricsResult result = scanReport.scanMetrics();
+ assertThat(result.totalPlanningDuration().totalDuration()).isGreaterThan(Duration.ZERO);
+ assertThat(result.resultDataFiles().value()).isEqualTo(3);
+ assertThat(result.resultDeleteFiles().value()).isEqualTo(2);
+ assertThat(result.scannedDataManifests().value()).isEqualTo(1);
+ assertThat(result.skippedDataManifests().value()).isEqualTo(0);
+ assertThat(result.totalDataManifests().value()).isEqualTo(1);
+ assertThat(result.totalDeleteManifests().value()).isEqualTo(1);
+ assertThat(result.totalFileSizeInBytes().value()).isEqualTo(30L);
+ assertThat(result.totalDeleteFileSizeInBytes().value()).isEqualTo(20L);
+ }
+
+ @Test
+ public void scanningWithSkippedDataFiles() throws IOException {
+ String tableName = "scan-planning-with-skipped-data-files";
+ Table table =
+ TestTables.create(
+ tableDir, tableName, SCHEMA, SPEC, SortOrder.unsorted(), formatVersion, reporter);
+ table.newAppend().appendFile(FILE_A).appendFile(FILE_D).commit();
+ table.newAppend().appendFile(FILE_B).appendFile(FILE_C).commit();
+ TableScan tableScan = table.newScan();
+
+ try (CloseableIterable<FileScanTask> fileScanTasks =
+ tableScan.filter(Expressions.equal("data", "1")).planFiles()) {
+ fileScanTasks.forEach(task -> {});
+ }
+
+ ScanReport scanReport = reporter.lastReport();
+ assertThat(scanReport).isNotNull();
+ assertThat(scanReport.tableName()).isEqualTo(tableName);
+ assertThat(scanReport.snapshotId()).isEqualTo(2L);
+ ScanMetricsResult result = scanReport.scanMetrics();
+ assertThat(result.skippedDataFiles().value()).isEqualTo(1);
+ assertThat(result.skippedDeleteFiles().value()).isEqualTo(0);
+ assertThat(result.totalPlanningDuration().totalDuration()).isGreaterThan(Duration.ZERO);
+ assertThat(result.resultDataFiles().value()).isEqualTo(1);
+ assertThat(result.resultDeleteFiles().value()).isEqualTo(0);
+ assertThat(result.scannedDataManifests().value()).isEqualTo(1);
+ assertThat(result.skippedDataManifests().value()).isEqualTo(1);
+ assertThat(result.totalDataManifests().value()).isEqualTo(2);
+ assertThat(result.totalDeleteManifests().value()).isEqualTo(0);
+ assertThat(result.totalFileSizeInBytes().value()).isEqualTo(10L);
+ assertThat(result.totalDeleteFileSizeInBytes().value()).isEqualTo(0L);
+ }
+
+ @Test
+ public void scanningWithSkippedDeleteFiles() throws IOException {
+ String tableName = "scan-planning-with-skipped-delete-files";
+ Table table =
+ TestTables.create(
+ tableDir, tableName, SCHEMA, SPEC, SortOrder.unsorted(), formatVersion, reporter);
+ table.newAppend().appendFile(FILE_A).appendFile(FILE_D).commit();
+ table.newRowDelta().addDeletes(FILE_A_DELETES).addDeletes(FILE_D2_DELETES).commit();
+ TableScan tableScan = table.newScan();
+
+ try (CloseableIterable<FileScanTask> fileScanTasks =
+ tableScan.filter(Expressions.equal("data", "1")).planFiles()) {
+ fileScanTasks.forEach(task -> {});
+ }
+
+ ScanReport scanReport = reporter.lastReport();
+ assertThat(scanReport).isNotNull();
+ assertThat(scanReport.tableName()).isEqualTo(tableName);
+ assertThat(scanReport.snapshotId()).isEqualTo(2L);
+ ScanMetricsResult result = scanReport.scanMetrics();
+ assertThat(result.totalPlanningDuration().totalDuration()).isGreaterThan(Duration.ZERO);
+ assertThat(result.resultDataFiles().value()).isEqualTo(1);
+ assertThat(result.resultDeleteFiles().value()).isEqualTo(1);
+ assertThat(result.skippedDataFiles().value()).isEqualTo(1);
+ assertThat(result.skippedDeleteFiles().value()).isEqualTo(1);
+ assertThat(result.scannedDataManifests().value()).isEqualTo(1);
+ assertThat(result.skippedDataManifests().value()).isEqualTo(0);
+ assertThat(result.totalDataManifests().value()).isEqualTo(1);
+ assertThat(result.totalDeleteManifests().value()).isEqualTo(1);
+ assertThat(result.totalFileSizeInBytes().value()).isEqualTo(10L);
+ assertThat(result.totalDeleteFileSizeInBytes().value()).isEqualTo(10L);
}
private static class TestScanReporter implements ScanReporter {
diff --git a/core/src/test/java/org/apache/iceberg/metrics/TestScanMetricsResultParser.java b/core/src/test/java/org/apache/iceberg/metrics/TestScanMetricsResultParser.java
index 57ec5a1bbe..00489cb0d3 100644
--- a/core/src/test/java/org/apache/iceberg/metrics/TestScanMetricsResultParser.java
+++ b/core/src/test/java/org/apache/iceberg/metrics/TestScanMetricsResultParser.java
@@ -42,6 +42,7 @@ public class TestScanMetricsResultParser {
.hasMessage("Invalid scan metrics: null");
}
+ @SuppressWarnings("MethodLength")
@Test
public void missingFields() {
Assertions.assertThat(ScanMetricsResultParser.fromJson("{}"))
@@ -129,6 +130,36 @@ public class TestScanMetricsResultParser {
+ "\"skipped-data-manifests\":{\"unit\":\"count\",\"value\":5},"
+ "\"total-file-size-in-bytes\":{\"unit\":\"bytes\",\"value\":1069}}"))
.isEqualTo(scanMetricsResult);
+
+ scanMetricsResult =
+ scanMetricsResult.withTotalDeleteFileSizeInBytes(CounterResult.of(Unit.BYTES, 1023L));
+ Assertions.assertThat(
+ ScanMetricsResultParser.fromJson(
+ "{\"total-planning-duration\":{\"count\":3,\"time-unit\":\"hours\",\"total-duration\":10},"
+ + "\"result-data-files\":{\"unit\":\"count\",\"value\":5},"
+ + "\"result-delete-files\":{\"unit\":\"count\",\"value\":5},"
+ + "\"total-data-manifests\":{\"unit\":\"count\",\"value\":5},"
+ + "\"total-delete-manifests\":{\"unit\":\"count\",\"value\":0},"
+ + "\"scanned-data-manifests\":{\"unit\":\"count\",\"value\":5},"
+ + "\"skipped-data-manifests\":{\"unit\":\"count\",\"value\":5},"
+ + "\"total-file-size-in-bytes\":{\"unit\":\"bytes\",\"value\":1069},"
+ + "\"total-delete-file-size-in-bytes\":{\"unit\":\"bytes\",\"value\":1023}}"))
+ .isEqualTo(scanMetricsResult);
+
+ scanMetricsResult = scanMetricsResult.withSkippedDataFiles(CounterResult.of(Unit.COUNT, 23L));
+ Assertions.assertThat(
+ ScanMetricsResultParser.fromJson(
+ "{\"total-planning-duration\":{\"count\":3,\"time-unit\":\"hours\",\"total-duration\":10},"
+ + "\"result-data-files\":{\"unit\":\"count\",\"value\":5},"
+ + "\"result-delete-files\":{\"unit\":\"count\",\"value\":5},"
+ + "\"total-data-manifests\":{\"unit\":\"count\",\"value\":5},"
+ + "\"total-delete-manifests\":{\"unit\":\"count\",\"value\":0},"
+ + "\"scanned-data-manifests\":{\"unit\":\"count\",\"value\":5},"
+ + "\"skipped-data-manifests\":{\"unit\":\"count\",\"value\":5},"
+ + "\"total-file-size-in-bytes\":{\"unit\":\"bytes\",\"value\":1069},"
+ + "\"total-delete-file-size-in-bytes\":{\"unit\":\"bytes\",\"value\":1023},"
+ + "\"skipped-data-files\":{\"unit\":\"count\",\"value\":23}}"))
+ .isEqualTo(scanMetricsResult);
}
@Test
@@ -143,6 +174,8 @@ public class TestScanMetricsResultParser {
scanMetrics.totalDataManifests().increment(5L);
scanMetrics.totalFileSizeInBytes().increment(45L);
scanMetrics.totalDeleteFileSizeInBytes().increment(23L);
+ scanMetrics.skippedDataFiles().increment(3L);
+ scanMetrics.skippedDeleteFiles().increment(3L);
ScanMetricsResult scanMetricsResult = ScanMetricsResult.fromScanMetrics(scanMetrics);
Assertions.assertThat(
@@ -156,6 +189,8 @@ public class TestScanMetricsResultParser {
+ "\"skipped-data-manifests\":{\"unit\":\"count\",\"value\":5},"
+ "\"total-file-size-in-bytes\":{\"unit\":\"bytes\",\"value\":1069},"
+ "\"total-delete-file-size-in-bytes\":{\"unit\":\"bytes\",\"value\":23},"
+ + "\"skipped-data-files\":{\"unit\":\"count\",\"value\":3},"
+ + "\"skipped-delete-files\":{\"unit\":\"count\",\"value\":3},"
+ "\"extra\": \"value\",\"extra2\":23}"))
.isEqualTo(scanMetricsResult);
}
@@ -193,6 +228,8 @@ public class TestScanMetricsResultParser {
scanMetrics.totalDataManifests().increment(5L);
scanMetrics.totalFileSizeInBytes().increment(45L);
scanMetrics.totalDeleteFileSizeInBytes().increment(23L);
+ scanMetrics.skippedDataFiles().increment(3L);
+ scanMetrics.skippedDeleteFiles().increment(3L);
ScanMetricsResult scanMetricsResult = ScanMetricsResult.fromScanMetrics(scanMetrics);
@@ -234,6 +271,14 @@ public class TestScanMetricsResultParser {
+ " \"total-delete-file-size-in-bytes\" : {\n"
+ " \"unit\" : \"bytes\",\n"
+ " \"value\" : 23\n"
+ + " },\n"
+ + " \"skipped-data-files\" : {\n"
+ + " \"unit\" : \"count\",\n"
+ + " \"value\" : 3\n"
+ + " },\n"
+ + " \"skipped-delete-files\" : {\n"
+ + " \"unit\" : \"count\",\n"
+ + " \"value\" : 3\n"
+ " }\n"
+ "}";
diff --git a/core/src/test/java/org/apache/iceberg/metrics/TestScanReportParser.java b/core/src/test/java/org/apache/iceberg/metrics/TestScanReportParser.java
index 3a669e175e..4666ec7068 100644
--- a/core/src/test/java/org/apache/iceberg/metrics/TestScanReportParser.java
+++ b/core/src/test/java/org/apache/iceberg/metrics/TestScanReportParser.java
@@ -80,6 +80,8 @@ public class TestScanReportParser {
scanMetrics.totalDataManifests().increment(5L);
scanMetrics.totalFileSizeInBytes().increment(45L);
scanMetrics.totalDeleteFileSizeInBytes().increment(23L);
+ scanMetrics.skippedDataFiles().increment(3L);
+ scanMetrics.skippedDeleteFiles().increment(3L);
String tableName = "roundTripTableName";
Schema projection =
@@ -106,6 +108,8 @@ public class TestScanReportParser {
+ "\"skipped-data-manifests\":{\"unit\":\"count\",\"value\":5},"
+ "\"total-file-size-in-bytes\":{\"unit\":\"bytes\",\"value\":1069},"
+ "\"total-delete-file-size-in-bytes\":{\"unit\":\"bytes\",\"value\":23},"
+ + "\"skipped-data-files\":{\"unit\":\"count\",\"value\":3},"
+ + "\"skipped-delete-files\":{\"unit\":\"count\",\"value\":3},"
+ "\"extra-metric\":\"extra-val\"},"
+ "\"extra\":\"extraVal\"}"))
.usingRecursiveComparison()
@@ -162,6 +166,8 @@ public class TestScanReportParser {
scanMetrics.totalDataManifests().increment(5L);
scanMetrics.totalFileSizeInBytes().increment(45L);
scanMetrics.totalDeleteFileSizeInBytes().increment(23L);
+ scanMetrics.skippedDataFiles().increment(3L);
+ scanMetrics.skippedDeleteFiles().increment(3L);
String tableName = "roundTripTableName";
Schema projection =
@@ -228,6 +234,14 @@ public class TestScanReportParser {
+ " \"total-delete-file-size-in-bytes\" : {\n"
+ " \"unit\" : \"bytes\",\n"
+ " \"value\" : 23\n"
+ + " },\n"
+ + " \"skipped-data-files\" : {\n"
+ + " \"unit\" : \"count\",\n"
+ + " \"value\" : 3\n"
+ + " },\n"
+ + " \"skipped-delete-files\" : {\n"
+ + " \"unit\" : \"count\",\n"
+ + " \"value\" : 3\n"
+ " }\n"
+ " }\n"
+ "}";