You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2022/10/27 22:03:52 UTC
[iceberg] branch master updated: Core: Replace projected Schema with schemaId/fieldIds/fieldNames in ScanReport (#6047)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 67be06d9c6 Core: Replace projected Schema with schemaId/fieldIds/fieldNames in ScanReport (#6047)
67be06d9c6 is described below
commit 67be06d9c6789f5b258db0b554a5dd96e645bc0c
Author: Eduard Tudenhöfner <et...@gmail.com>
AuthorDate: Fri Oct 28 00:03:45 2022 +0200
Core: Replace projected Schema with schemaId/fieldIds/fieldNames in ScanReport (#6047)
The motivation behind this change is that the projected schema might get
quite big and contain information such as doc comments, which make it
quite hard to read/consume. This change makes sure that we only include
the minimal set of information from a schema
(schemaId/fieldIds/fieldNames).
---
.../java/org/apache/iceberg/BaseTableScan.java | 12 ++-
.../org/apache/iceberg/metrics/ScanReport.java | 8 +-
.../apache/iceberg/metrics/ScanReportParser.java | 30 ++++--
.../org/apache/iceberg/metrics/TestScanReport.java | 44 ++++----
.../iceberg/metrics/TestScanReportParser.java | 112 ++++++++++++---------
.../requests/TestReportMetricsRequestParser.java | 24 ++---
open-api/rest-catalog-open-api.yaml | 96 ++++++++++--------
7 files changed, 190 insertions(+), 136 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/BaseTableScan.java b/core/src/main/java/org/apache/iceberg/BaseTableScan.java
index 47dbe9c7a3..fcb432c410 100644
--- a/core/src/main/java/org/apache/iceberg/BaseTableScan.java
+++ b/core/src/main/java/org/apache/iceberg/BaseTableScan.java
@@ -18,7 +18,9 @@
*/
package org.apache.iceberg;
+import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.iceberg.events.Listeners;
import org.apache.iceberg.events.ScanEvent;
import org.apache.iceberg.expressions.ExpressionUtil;
@@ -31,6 +33,8 @@ import org.apache.iceberg.metrics.ScanReport;
import org.apache.iceberg.metrics.Timer;
import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.TypeUtil;
import org.apache.iceberg.util.DateTimeUtil;
import org.apache.iceberg.util.SnapshotUtil;
import org.apache.iceberg.util.TableScanUtil;
@@ -127,6 +131,10 @@ abstract class BaseTableScan extends BaseScan<TableScan, FileScanTask, CombinedS
ExpressionUtil.toSanitizedString(filter()));
Listeners.notifyAll(new ScanEvent(table().name(), snapshot.snapshotId(), filter(), schema()));
+ List<Integer> projectedFieldIds = Lists.newArrayList(TypeUtil.getProjectedIds(schema()));
+ List<String> projectedFieldNames =
+ projectedFieldIds.stream().map(schema()::findColumnName).collect(Collectors.toList());
+
Timer.Timed planningDuration = scanMetrics().totalPlanningDuration().start();
return CloseableIterable.whenComplete(
@@ -135,7 +143,9 @@ abstract class BaseTableScan extends BaseScan<TableScan, FileScanTask, CombinedS
planningDuration.stop();
ScanReport scanReport =
ImmutableScanReport.builder()
- .projection(schema())
+ .schemaId(schema().schemaId())
+ .projectedFieldIds(projectedFieldIds)
+ .projectedFieldNames(projectedFieldNames)
.tableName(table().name())
.snapshotId(snapshot.snapshotId())
.filter(ExpressionUtil.sanitize(filter()))
diff --git a/core/src/main/java/org/apache/iceberg/metrics/ScanReport.java b/core/src/main/java/org/apache/iceberg/metrics/ScanReport.java
index be1d0d18de..451dbbaf76 100644
--- a/core/src/main/java/org/apache/iceberg/metrics/ScanReport.java
+++ b/core/src/main/java/org/apache/iceberg/metrics/ScanReport.java
@@ -18,7 +18,7 @@
*/
package org.apache.iceberg.metrics;
-import org.apache.iceberg.Schema;
+import java.util.List;
import org.apache.iceberg.expressions.Expression;
import org.immutables.value.Value;
@@ -32,7 +32,11 @@ public interface ScanReport extends MetricsReport {
Expression filter();
- Schema projection();
+ int schemaId();
+
+ List<Integer> projectedFieldIds();
+
+ List<String> projectedFieldNames();
ScanMetricsResult scanMetrics();
}
diff --git a/core/src/main/java/org/apache/iceberg/metrics/ScanReportParser.java b/core/src/main/java/org/apache/iceberg/metrics/ScanReportParser.java
index 05138e1a20..1331d88242 100644
--- a/core/src/main/java/org/apache/iceberg/metrics/ScanReportParser.java
+++ b/core/src/main/java/org/apache/iceberg/metrics/ScanReportParser.java
@@ -21,8 +21,7 @@ package org.apache.iceberg.metrics;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.IOException;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.SchemaParser;
+import java.util.List;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.ExpressionParser;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
@@ -32,7 +31,9 @@ public class ScanReportParser {
private static final String TABLE_NAME = "table-name";
private static final String SNAPSHOT_ID = "snapshot-id";
private static final String FILTER = "filter";
- private static final String PROJECTION = "projection";
+ private static final String SCHEMA_ID = "schema-id";
+ private static final String PROJECTED_FIELD_IDS = "projected-field-ids";
+ private static final String PROJECTED_FIELD_NAMES = "projected-field-names";
private static final String METRICS = "metrics";
private ScanReportParser() {}
@@ -71,8 +72,19 @@ public class ScanReportParser {
gen.writeFieldName(FILTER);
ExpressionParser.toJson(scanReport.filter(), gen);
- gen.writeFieldName(PROJECTION);
- SchemaParser.toJson(scanReport.projection(), gen);
+ gen.writeNumberField(SCHEMA_ID, scanReport.schemaId());
+
+ gen.writeArrayFieldStart(PROJECTED_FIELD_IDS);
+ for (Integer fieldId : scanReport.projectedFieldIds()) {
+ gen.writeNumber(fieldId);
+ }
+ gen.writeEndArray();
+
+ gen.writeArrayFieldStart(PROJECTED_FIELD_NAMES);
+ for (String fieldName : scanReport.projectedFieldNames()) {
+ gen.writeString(fieldName);
+ }
+ gen.writeEndArray();
gen.writeFieldName(METRICS);
ScanMetricsResultParser.toJson(scanReport.scanMetrics(), gen);
@@ -90,13 +102,17 @@ public class ScanReportParser {
String tableName = JsonUtil.getString(TABLE_NAME, json);
long snapshotId = JsonUtil.getLong(SNAPSHOT_ID, json);
Expression filter = ExpressionParser.fromJson(JsonUtil.get(FILTER, json));
- Schema projection = SchemaParser.fromJson(JsonUtil.get(PROJECTION, json));
+ int schemaId = JsonUtil.getInt(SCHEMA_ID, json);
+ List<Integer> projectedFieldIds = JsonUtil.getIntegerList(PROJECTED_FIELD_IDS, json);
+ List<String> projectedFieldNames = JsonUtil.getStringList(PROJECTED_FIELD_NAMES, json);
ScanMetricsResult scanMetricsResult =
ScanMetricsResultParser.fromJson(JsonUtil.get(METRICS, json));
return ImmutableScanReport.builder()
.tableName(tableName)
.snapshotId(snapshotId)
- .projection(projection)
+ .schemaId(schemaId)
+ .projectedFieldIds(projectedFieldIds)
+ .projectedFieldNames(projectedFieldNames)
.filter(filter)
.scanMetrics(scanMetricsResult)
.build();
diff --git a/core/src/test/java/org/apache/iceberg/metrics/TestScanReport.java b/core/src/test/java/org/apache/iceberg/metrics/TestScanReport.java
index 3cfadcc61c..ab299b921d 100644
--- a/core/src/test/java/org/apache/iceberg/metrics/TestScanReport.java
+++ b/core/src/test/java/org/apache/iceberg/metrics/TestScanReport.java
@@ -19,10 +19,10 @@
package org.apache.iceberg.metrics;
import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
import java.util.concurrent.TimeUnit;
-import org.apache.iceberg.Schema;
import org.apache.iceberg.expressions.Expressions;
-import org.apache.iceberg.types.Types;
import org.assertj.core.api.Assertions;
import org.junit.Test;
@@ -33,12 +33,12 @@ public class TestScanReport {
Assertions.assertThatThrownBy(() -> ImmutableScanReport.builder().build())
.isInstanceOf(IllegalStateException.class)
.hasMessage(
- "Cannot build ScanReport, some of required attributes are not set [tableName, snapshotId, filter, projection, scanMetrics]");
+ "Cannot build ScanReport, some of required attributes are not set [tableName, snapshotId, filter, schemaId, scanMetrics]");
Assertions.assertThatThrownBy(() -> ImmutableScanReport.builder().tableName("x").build())
.isInstanceOf(IllegalStateException.class)
.hasMessage(
- "Cannot build ScanReport, some of required attributes are not set [snapshotId, filter, projection, scanMetrics]");
+ "Cannot build ScanReport, some of required attributes are not set [snapshotId, filter, schemaId, scanMetrics]");
Assertions.assertThatThrownBy(
() ->
@@ -48,7 +48,7 @@ public class TestScanReport {
.build())
.isInstanceOf(IllegalStateException.class)
.hasMessage(
- "Cannot build ScanReport, some of required attributes are not set [snapshotId, projection, scanMetrics]");
+ "Cannot build ScanReport, some of required attributes are not set [snapshotId, schemaId, scanMetrics]");
Assertions.assertThatThrownBy(
() ->
@@ -59,7 +59,7 @@ public class TestScanReport {
.build())
.isInstanceOf(IllegalStateException.class)
.hasMessage(
- "Cannot build ScanReport, some of required attributes are not set [projection, scanMetrics]");
+ "Cannot build ScanReport, some of required attributes are not set [schemaId, scanMetrics]");
Assertions.assertThatThrownBy(
() ->
@@ -67,9 +67,9 @@ public class TestScanReport {
.tableName("x")
.filter(Expressions.alwaysTrue())
.snapshotId(23L)
- .projection(
- new Schema(
- Types.NestedField.required(1, "c1", Types.StringType.get(), "c1")))
+ .schemaId(4)
+ .addProjectedFieldIds(1, 2)
+ .addProjectedFieldNames("c1", "c2")
.build())
.isInstanceOf(IllegalStateException.class)
.hasMessage(
@@ -79,19 +79,24 @@ public class TestScanReport {
@Test
public void fromEmptyScanMetrics() {
String tableName = "x";
- Schema projection =
- new Schema(Types.NestedField.required(1, "c1", Types.StringType.get(), "c1"));
+ int schemaId = 4;
+ List<Integer> fieldIds = Arrays.asList(1, 2);
+ List<String> fieldNames = Arrays.asList("c1", "c2");
ScanReport scanReport =
ImmutableScanReport.builder()
.tableName(tableName)
.snapshotId(23L)
.filter(Expressions.alwaysTrue())
- .projection(projection)
+ .schemaId(schemaId)
+ .projectedFieldIds(fieldIds)
+ .projectedFieldNames(fieldNames)
.scanMetrics(ScanMetricsResult.fromScanMetrics(ScanMetrics.noop()))
.build();
Assertions.assertThat(scanReport.tableName()).isEqualTo(tableName);
- Assertions.assertThat(scanReport.projection()).isEqualTo(projection);
+ Assertions.assertThat(scanReport.schemaId()).isEqualTo(schemaId);
+ Assertions.assertThat(scanReport.projectedFieldIds()).isEqualTo(fieldIds);
+ Assertions.assertThat(scanReport.projectedFieldNames()).isEqualTo(fieldNames);
Assertions.assertThat(scanReport.filter()).isEqualTo(Expressions.alwaysTrue());
Assertions.assertThat(scanReport.snapshotId()).isEqualTo(23L);
Assertions.assertThat(scanReport.scanMetrics().totalPlanningDuration()).isNull();
@@ -116,20 +121,25 @@ public class TestScanReport {
scanMetrics.totalDataManifests().increment(5L);
String tableName = "x";
- Schema projection =
- new Schema(Types.NestedField.required(1, "c1", Types.StringType.get(), "c1"));
+ int schemaId = 4;
+ List<Integer> fieldIds = Arrays.asList(1, 2);
+ List<String> fieldNames = Arrays.asList("c1", "c2");
ScanReport scanReport =
ImmutableScanReport.builder()
.tableName(tableName)
.snapshotId(23L)
.filter(Expressions.alwaysTrue())
- .projection(projection)
+ .schemaId(schemaId)
+ .projectedFieldIds(fieldIds)
+ .projectedFieldNames(fieldNames)
.scanMetrics(ScanMetricsResult.fromScanMetrics(scanMetrics))
.build();
Assertions.assertThat(scanReport.tableName()).isEqualTo(tableName);
- Assertions.assertThat(scanReport.projection()).isEqualTo(projection);
+ Assertions.assertThat(scanReport.schemaId()).isEqualTo(schemaId);
+ Assertions.assertThat(scanReport.projectedFieldIds()).isEqualTo(fieldIds);
+ Assertions.assertThat(scanReport.projectedFieldNames()).isEqualTo(fieldNames);
Assertions.assertThat(scanReport.filter()).isEqualTo(Expressions.alwaysTrue());
Assertions.assertThat(scanReport.snapshotId()).isEqualTo(23L);
Assertions.assertThat(scanReport.scanMetrics().totalPlanningDuration().totalDuration())
diff --git a/core/src/test/java/org/apache/iceberg/metrics/TestScanReportParser.java b/core/src/test/java/org/apache/iceberg/metrics/TestScanReportParser.java
index 0c0ff71773..9bdb6c9f49 100644
--- a/core/src/test/java/org/apache/iceberg/metrics/TestScanReportParser.java
+++ b/core/src/test/java/org/apache/iceberg/metrics/TestScanReportParser.java
@@ -20,9 +20,7 @@ package org.apache.iceberg.metrics;
import com.fasterxml.jackson.databind.JsonNode;
import java.util.concurrent.TimeUnit;
-import org.apache.iceberg.Schema;
import org.apache.iceberg.expressions.Expressions;
-import org.apache.iceberg.types.Types;
import org.assertj.core.api.Assertions;
import org.junit.Test;
@@ -55,13 +53,13 @@ public class TestScanReportParser {
ScanReportParser.fromJson(
"{\"table-name\":\"roundTripTableName\",\"snapshot-id\":23,\"filter\":true}"))
.isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Cannot parse missing field: projection");
+ .hasMessage("Cannot parse missing int: schema-id");
Assertions.assertThatThrownBy(
() ->
ScanReportParser.fromJson(
"{\"table-name\":\"roundTripTableName\",\"snapshot-id\":23,\"filter\":true,"
- + "\"projection\":{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"c1\",\"required\":true,\"type\":\"string\",\"doc\":\"c1\"}]}}"))
+ + "\"schema-id\" : 4,\"projected-field-ids\" : [ 1, 2, 3 ],\"projected-field-names\" : [ \"c1\", \"c2\", \"c3\" ]}"))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot parse missing field: metrics");
}
@@ -87,12 +85,12 @@ public class TestScanReportParser {
scanMetrics.equalityDeleteFiles().increment(4L);
String tableName = "roundTripTableName";
- Schema projection =
- new Schema(Types.NestedField.required(1, "c1", Types.StringType.get(), "c1"));
ScanReport scanReport =
ImmutableScanReport.builder()
.tableName(tableName)
- .projection(projection)
+ .schemaId(4)
+ .addProjectedFieldIds(1, 2, 3)
+ .addProjectedFieldNames("c1", "c2", "c3")
.snapshotId(23L)
.filter(Expressions.alwaysTrue())
.scanMetrics(ScanMetricsResult.fromScanMetrics(scanMetrics))
@@ -101,7 +99,7 @@ public class TestScanReportParser {
Assertions.assertThat(
ScanReportParser.fromJson(
"{\"table-name\":\"roundTripTableName\",\"snapshot-id\":23,"
- + "\"filter\":true,\"projection\":{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"c1\",\"required\":true,\"type\":\"string\",\"doc\":\"c1\"}]},"
+ + "\"filter\":true,\"schema-id\": 4,\"projected-field-ids\": [ 1, 2, 3 ],\"projected-field-names\": [ \"c1\", \"c2\", \"c3\" ],"
+ "\"metrics\":{\"total-planning-duration\":{\"count\":1,\"time-unit\":\"nanoseconds\",\"total-duration\":600000000000},"
+ "\"result-data-files\":{\"unit\":\"count\",\"value\":5},"
+ "\"result-delete-files\":{\"unit\":\"count\",\"value\":5},"
@@ -120,8 +118,6 @@ public class TestScanReportParser {
+ "\"positional-delete-files\":{\"unit\":\"count\",\"value\":6},"
+ "\"extra-metric\":\"extra-val\"},"
+ "\"extra\":\"extraVal\"}"))
- .usingRecursiveComparison()
- .ignoringFields("projection")
.isEqualTo(scanReport);
}
@@ -157,9 +153,23 @@ public class TestScanReportParser {
Assertions.assertThatThrownBy(
() ->
ScanReportParser.fromJson(
- "{\"table-name\":\"roundTripTableName\",\"snapshot-id\":23,\"filter\":true,\"projection\":23}"))
+ "{\"table-name\":\"roundTripTableName\",\"snapshot-id\":23,\"filter\":true,\"schema-id\":\"23\"}"))
.isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Cannot parse type from json: 23");
+ .hasMessage("Cannot parse to an integer value: schema-id: \"23\"");
+
+ Assertions.assertThatThrownBy(
+ () ->
+ ScanReportParser.fromJson(
+ "{\"table-name\":\"roundTripTableName\",\"snapshot-id\":23,\"filter\":true,\"schema-id\":23,\"projected-field-ids\": [\"1\"],\"metrics\":{}}"))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Cannot parse integer from non-int value: \"1\"");
+
+ Assertions.assertThatThrownBy(
+ () ->
+ ScanReportParser.fromJson(
+ "{\"table-name\":\"roundTripTableName\",\"snapshot-id\":23,\"filter\":true,\"schema-id\":23,\"projected-field-ids\": [1],\"projected-field-names\": [1],\"metrics\":{}}"))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Cannot parse string from non-text value: 1");
}
@Test
@@ -183,12 +193,12 @@ public class TestScanReportParser {
scanMetrics.equalityDeleteFiles().increment(4L);
String tableName = "roundTripTableName";
- Schema projection =
- new Schema(Types.NestedField.required(1, "c1", Types.StringType.get(), "c1"));
ScanReport scanReport =
ImmutableScanReport.builder()
.tableName(tableName)
- .projection(projection)
+ .schemaId(4)
+ .addProjectedFieldIds(1, 2, 3)
+ .addProjectedFieldNames("c1", "c2", "c3")
.filter(Expressions.alwaysTrue())
.snapshotId(23L)
.scanMetrics(ScanMetricsResult.fromScanMetrics(scanMetrics))
@@ -199,17 +209,9 @@ public class TestScanReportParser {
+ " \"table-name\" : \"roundTripTableName\",\n"
+ " \"snapshot-id\" : 23,\n"
+ " \"filter\" : true,\n"
- + " \"projection\" : {\n"
- + " \"type\" : \"struct\",\n"
- + " \"schema-id\" : 0,\n"
- + " \"fields\" : [ {\n"
- + " \"id\" : 1,\n"
- + " \"name\" : \"c1\",\n"
- + " \"required\" : true,\n"
- + " \"type\" : \"string\",\n"
- + " \"doc\" : \"c1\"\n"
- + " } ]\n"
- + " },\n"
+ + " \"schema-id\" : 4,\n"
+ + " \"projected-field-ids\" : [ 1, 2, 3 ],\n"
+ + " \"projected-field-names\" : [ \"c1\", \"c2\", \"c3\" ],\n"
+ " \"metrics\" : {\n"
+ " \"total-planning-duration\" : {\n"
+ " \"count\" : 1,\n"
@@ -280,22 +282,19 @@ public class TestScanReportParser {
+ "}";
String json = ScanReportParser.toJson(scanReport, true);
- Assertions.assertThat(ScanReportParser.fromJson(json))
- .usingRecursiveComparison()
- .ignoringFields("projection")
- .isEqualTo(scanReport);
+ Assertions.assertThat(ScanReportParser.fromJson(json)).isEqualTo(scanReport);
Assertions.assertThat(json).isEqualTo(expectedJson);
}
@Test
public void roundTripSerdeWithNoopMetrics() {
String tableName = "roundTripTableName";
- Schema projection =
- new Schema(Types.NestedField.required(1, "c1", Types.StringType.get(), "c1"));
ScanReport scanReport =
ImmutableScanReport.builder()
.tableName(tableName)
- .projection(projection)
+ .schemaId(4)
+ .addProjectedFieldIds(1, 2, 3)
+ .addProjectedFieldNames("c1", "c2", "c3")
.snapshotId(23L)
.filter(Expressions.alwaysTrue())
.scanMetrics(ScanMetricsResult.fromScanMetrics(ScanMetrics.noop()))
@@ -306,25 +305,42 @@ public class TestScanReportParser {
+ " \"table-name\" : \"roundTripTableName\",\n"
+ " \"snapshot-id\" : 23,\n"
+ " \"filter\" : true,\n"
- + " \"projection\" : {\n"
- + " \"type\" : \"struct\",\n"
- + " \"schema-id\" : 0,\n"
- + " \"fields\" : [ {\n"
- + " \"id\" : 1,\n"
- + " \"name\" : \"c1\",\n"
- + " \"required\" : true,\n"
- + " \"type\" : \"string\",\n"
- + " \"doc\" : \"c1\"\n"
- + " } ]\n"
- + " },\n"
+ + " \"schema-id\" : 4,\n"
+ + " \"projected-field-ids\" : [ 1, 2, 3 ],\n"
+ + " \"projected-field-names\" : [ \"c1\", \"c2\", \"c3\" ],\n"
+ " \"metrics\" : { }\n"
+ "}";
String json = ScanReportParser.toJson(scanReport, true);
- Assertions.assertThat(ScanReportParser.fromJson(json))
- .usingRecursiveComparison()
- .ignoringFields("projection")
- .isEqualTo(scanReport);
+ Assertions.assertThat(ScanReportParser.fromJson(json)).isEqualTo(scanReport);
+ Assertions.assertThat(json).isEqualTo(expectedJson);
+ }
+
+ @Test
+ public void roundTripSerdeWithEmptyFieldIdsAndNames() {
+ String tableName = "roundTripTableName";
+ ScanReport scanReport =
+ ImmutableScanReport.builder()
+ .tableName(tableName)
+ .schemaId(4)
+ .snapshotId(23L)
+ .filter(Expressions.alwaysTrue())
+ .scanMetrics(ScanMetricsResult.fromScanMetrics(ScanMetrics.noop()))
+ .build();
+
+ String expectedJson =
+ "{\n"
+ + " \"table-name\" : \"roundTripTableName\",\n"
+ + " \"snapshot-id\" : 23,\n"
+ + " \"filter\" : true,\n"
+ + " \"schema-id\" : 4,\n"
+ + " \"projected-field-ids\" : [ ],\n"
+ + " \"projected-field-names\" : [ ],\n"
+ + " \"metrics\" : { }\n"
+ + "}";
+
+ String json = ScanReportParser.toJson(scanReport, true);
+ Assertions.assertThat(ScanReportParser.fromJson(json)).isEqualTo(scanReport);
Assertions.assertThat(json).isEqualTo(expectedJson);
}
}
diff --git a/core/src/test/java/org/apache/iceberg/rest/requests/TestReportMetricsRequestParser.java b/core/src/test/java/org/apache/iceberg/rest/requests/TestReportMetricsRequestParser.java
index a6d0b6b39d..a97bdfe3bc 100644
--- a/core/src/test/java/org/apache/iceberg/rest/requests/TestReportMetricsRequestParser.java
+++ b/core/src/test/java/org/apache/iceberg/rest/requests/TestReportMetricsRequestParser.java
@@ -19,14 +19,12 @@
package org.apache.iceberg.rest.requests;
import com.fasterxml.jackson.databind.JsonNode;
-import org.apache.iceberg.Schema;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.metrics.ImmutableScanReport;
import org.apache.iceberg.metrics.MetricsReport;
import org.apache.iceberg.metrics.ScanMetrics;
import org.apache.iceberg.metrics.ScanMetricsResult;
import org.apache.iceberg.metrics.ScanReport;
-import org.apache.iceberg.types.Types;
import org.assertj.core.api.Assertions;
import org.junit.Test;
@@ -82,12 +80,12 @@ public class TestReportMetricsRequestParser {
@Test
public void roundTripSerde() {
String tableName = "roundTripTableName";
- Schema projection =
- new Schema(Types.NestedField.required(1, "c1", Types.StringType.get(), "c1"));
ScanReport scanReport =
ImmutableScanReport.builder()
.tableName(tableName)
- .projection(projection)
+ .schemaId(4)
+ .addProjectedFieldIds(1, 2, 3)
+ .addProjectedFieldNames("c1", "c2", "c3")
.snapshotId(23L)
.filter(Expressions.alwaysTrue())
.scanMetrics(ScanMetricsResult.fromScanMetrics(ScanMetrics.noop()))
@@ -99,17 +97,9 @@ public class TestReportMetricsRequestParser {
+ " \"table-name\" : \"roundTripTableName\",\n"
+ " \"snapshot-id\" : 23,\n"
+ " \"filter\" : true,\n"
- + " \"projection\" : {\n"
- + " \"type\" : \"struct\",\n"
- + " \"schema-id\" : 0,\n"
- + " \"fields\" : [ {\n"
- + " \"id\" : 1,\n"
- + " \"name\" : \"c1\",\n"
- + " \"required\" : true,\n"
- + " \"type\" : \"string\",\n"
- + " \"doc\" : \"c1\"\n"
- + " } ]\n"
- + " },\n"
+ + " \"schema-id\" : 4,\n"
+ + " \"projected-field-ids\" : [ 1, 2, 3 ],\n"
+ + " \"projected-field-names\" : [ \"c1\", \"c2\", \"c3\" ],\n"
+ " \"metrics\" : { }\n"
+ "}";
@@ -119,8 +109,6 @@ public class TestReportMetricsRequestParser {
Assertions.assertThat(json).isEqualTo(expectedJson);
Assertions.assertThat(ReportMetricsRequestParser.fromJson(json).report())
- .usingRecursiveComparison()
- .ignoringFields("projection")
.isEqualTo(metricsRequest.report());
}
}
diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml
index 991eea3f90..d95b533683 100644
--- a/open-api/rest-catalog-open-api.yaml
+++ b/open-api/rest-catalog-open-api.yaml
@@ -1837,6 +1837,46 @@ components:
type: object
additionalProperties:
$ref: '#/components/schemas/MetricResult'
+ example:
+ "metrics": {
+ "total-planning-duration": {
+ "count": 1,
+ "time-unit": "nanoseconds",
+ "total-duration": 2644235116
+ },
+ "result-data-files": {
+ "unit": "count",
+ "value": 1,
+ },
+ "result-delete-files": {
+ "unit": "count",
+ "value": 0,
+ },
+ "total-data-manifests": {
+ "unit": "count",
+ "value": 1,
+ },
+ "total-delete-manifests": {
+ "unit": "count",
+ "value": 0,
+ },
+ "scanned-data-manifests": {
+ "unit": "count",
+ "value": 1,
+ },
+ "skipped-data-manifests": {
+ "unit": "count",
+ "value": 0,
+ },
+ "total-file-size-bytes": {
+ "unit": "bytes",
+ "value": 10,
+ },
+ "total-delete-file-size-bytes": {
+ "unit": "bytes",
+ "value": 0,
+ }
+ }
ReportMetricsRequest:
anyOf:
@@ -1853,7 +1893,9 @@ components:
- table-name
- snapshot-id
- filter
- - projection
+ - schema-id
+ - projected-field-ids
+ - projected-field-names
- metrics
properties:
table-name:
@@ -1863,50 +1905,18 @@ components:
format: int64
filter:
$ref: '#/components/schemas/Expression'
- projection:
- $ref: '#/components/schemas/Schema'
+ schema-id:
+ type: integer
+ projected-field-ids:
+ type: array
+ items:
+ type: integer
+ projected-field-names:
+ type: array
+ items:
+ type: string
metrics:
$ref: '#/components/schemas/Metrics'
- example:
- "metrics": {
- "total-planning-duration": {
- "count": 1,
- "time-unit": "nanoseconds",
- "total-duration": 2644235116
- },
- "result-data-files": {
- "unit": "count",
- "value": 1,
- },
- "result-delete-files": {
- "unit": "count",
- "value": 0,
- },
- "total-data-manifests": {
- "unit": "count",
- "value": 1,
- },
- "total-delete-manifests": {
- "unit": "count",
- "value": 0,
- },
- "scanned-data-manifests": {
- "unit": "count",
- "value": 1,
- },
- "skipped-data-manifests": {
- "unit": "count",
- "value": 0,
- },
- "total-file-size-bytes": {
- "unit": "bytes",
- "value": 10,
- },
- "total-delete-file-size-bytes": {
- "unit": "bytes",
- "value": 0,
- }
- }
#############################