You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by sz...@apache.org on 2026/08/28 01:13:00 UTC
(iceberg) branch main updated: API, AWS, Azure, GCP: Track read metrics for positioned, vectored, and accelerated reads (#17236)
This is an automated email from the ASF dual-hosted git repository.
szehon-ho pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new ef8a69dc3d API, AWS, Azure, GCP: Track read metrics for positioned, vectored, and accelerated reads (#17236)
ef8a69dc3d is described below
commit ef8a69dc3dab5858d612770062bbf5a6043170f0
Author: DB Tsai <db...@dbtsai.com>
AuthorDate: Thu Aug 27 18:12:53 2026 -0700
API, AWS, Azure, GCP: Track read metrics for positioned, vectored, and accelerated reads (#17236)
---
.../iceberg/metrics/CachingMetricsContext.java | 32 +++++
.../s3/AnalyticsAcceleratorInputStreamWrapper.java | 26 +++-
.../iceberg/aws/s3/AnalyticsAcceleratorUtil.java | 2 +-
.../org/apache/iceberg/aws/s3/S3InputStream.java | 17 ++-
...TestAnalyticsAcceleratorInputStreamWrapper.java | 72 ++++++++++
.../apache/iceberg/aws/s3/TestS3InputStream.java | 92 ++++++++++++-
.../iceberg/azure/adlsv2/ADLSInputStream.java | 13 +-
.../iceberg/azure/adlsv2/TestADLSInputStream.java | 91 ++++++++++++
.../apache/iceberg/gcp/gcs/AnalyticsCoreUtil.java | 36 ++++-
.../org/apache/iceberg/gcp/gcs/GCSInputStream.java | 11 +-
.../iceberg/gcp/gcs/TestAnalyticsCoreUtil.java | 152 +++++++++++++++++++++
.../apache/iceberg/gcp/gcs/TestGCSInputStream.java | 46 +++++++
12 files changed, 572 insertions(+), 18 deletions(-)
diff --git a/api/src/test/java/org/apache/iceberg/metrics/CachingMetricsContext.java b/api/src/test/java/org/apache/iceberg/metrics/CachingMetricsContext.java
new file mode 100644
index 0000000000..c3b46134a9
--- /dev/null
+++ b/api/src/test/java/org/apache/iceberg/metrics/CachingMetricsContext.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.metrics;
+
+import java.util.Map;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A {@link MetricsContext} that hands out the same counter per name so tests can read it back. */
+public class CachingMetricsContext extends DefaultMetricsContext {
+ private final Map<String, org.apache.iceberg.metrics.Counter> counters = Maps.newConcurrentMap();
+
+ @Override
+ public org.apache.iceberg.metrics.Counter counter(String name, Unit unit) {
+ return counters.computeIfAbsent(name, ignored -> super.counter(name, unit));
+ }
+}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/AnalyticsAcceleratorInputStreamWrapper.java b/aws/src/main/java/org/apache/iceberg/aws/s3/AnalyticsAcceleratorInputStreamWrapper.java
index 197f79905f..ae8a315866 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/AnalyticsAcceleratorInputStreamWrapper.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/AnalyticsAcceleratorInputStreamWrapper.java
@@ -19,31 +19,49 @@
package org.apache.iceberg.aws.s3;
import java.io.IOException;
+import org.apache.iceberg.io.FileIOMetricsContext;
import org.apache.iceberg.io.SeekableInputStream;
+import org.apache.iceberg.metrics.Counter;
+import org.apache.iceberg.metrics.MetricsContext;
+import org.apache.iceberg.metrics.MetricsContext.Unit;
import software.amazon.s3.analyticsaccelerator.S3SeekableInputStream;
/** A wrapper to convert {@link S3SeekableInputStream} to Iceberg {@link SeekableInputStream} */
class AnalyticsAcceleratorInputStreamWrapper extends SeekableInputStream {
private final S3SeekableInputStream delegate;
+ private final Counter readBytes;
+ private final Counter readOperations;
- AnalyticsAcceleratorInputStreamWrapper(S3SeekableInputStream stream) {
+ AnalyticsAcceleratorInputStreamWrapper(S3SeekableInputStream stream, MetricsContext metrics) {
this.delegate = stream;
+ this.readBytes = metrics.counter(FileIOMetricsContext.READ_BYTES, Unit.BYTES);
+ this.readOperations = metrics.counter(FileIOMetricsContext.READ_OPERATIONS);
}
@Override
public int read() throws IOException {
- return this.delegate.read();
+ int bytesRead = this.delegate.read();
+ if (bytesRead != -1) {
+ readBytes.increment();
+ readOperations.increment();
+ }
+ return bytesRead;
}
@Override
public int read(byte[] b) throws IOException {
- return this.delegate.read(b, 0, b.length);
+ return read(b, 0, b.length);
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
- return this.delegate.read(b, off, len);
+ int bytesRead = this.delegate.read(b, off, len);
+ if (bytesRead > 0) {
+ readBytes.increment(bytesRead);
+ readOperations.increment();
+ }
+ return bytesRead;
}
@Override
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/AnalyticsAcceleratorUtil.java b/aws/src/main/java/org/apache/iceberg/aws/s3/AnalyticsAcceleratorUtil.java
index 36a0a235da..b6886d9f58 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/AnalyticsAcceleratorUtil.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/AnalyticsAcceleratorUtil.java
@@ -63,7 +63,7 @@ class AnalyticsAcceleratorUtil {
try {
S3SeekableInputStream seekableInputStream = factory.createStream(uri);
- return new AnalyticsAcceleratorInputStreamWrapper(seekableInputStream);
+ return new AnalyticsAcceleratorInputStreamWrapper(seekableInputStream, inputFile.metrics());
} catch (IOException e) {
throw new RuntimeIOException(
e, "Failed to create S3 analytics accelerator input stream for: %s", inputFile.uri());
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/S3InputStream.java b/aws/src/main/java/org/apache/iceberg/aws/s3/S3InputStream.java
index ef889d2363..ae0a4498a4 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/S3InputStream.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3InputStream.java
@@ -173,8 +173,12 @@ class S3InputStream extends SeekableInputStream implements RangeReadable {
String range = String.format("bytes=%s-%s", position, position + length - 1);
- try (InputStream stream = readRange(range)) {
- IOUtil.readFully(stream, buffer, offset, length);
+ try (InputStream rangeStream = readRange(range)) {
+ IOUtil.readFully(rangeStream, buffer, offset, length);
+ if (length > 0) {
+ readBytes.increment(length);
+ readOperations.increment();
+ }
}
}
@@ -184,8 +188,13 @@ class S3InputStream extends SeekableInputStream implements RangeReadable {
String range = String.format("bytes=-%s", length);
- try (InputStream stream = readRange(range)) {
- return IOUtil.readRemaining(stream, buffer, offset, length);
+ try (InputStream rangeStream = readRange(range)) {
+ int bytesRead = IOUtil.readRemaining(rangeStream, buffer, offset, length);
+ if (bytesRead > 0) {
+ readBytes.increment(bytesRead);
+ readOperations.increment();
+ }
+ return bytesRead;
}
}
diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/TestAnalyticsAcceleratorInputStreamWrapper.java b/aws/src/test/java/org/apache/iceberg/aws/s3/TestAnalyticsAcceleratorInputStreamWrapper.java
new file mode 100644
index 0000000000..a457a1e408
--- /dev/null
+++ b/aws/src/test/java/org/apache/iceberg/aws/s3/TestAnalyticsAcceleratorInputStreamWrapper.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.aws.s3;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import org.apache.iceberg.io.FileIOMetricsContext;
+import org.apache.iceberg.metrics.CachingMetricsContext;
+import org.apache.iceberg.metrics.Counter;
+import org.apache.iceberg.metrics.MetricsContext;
+import org.junit.jupiter.api.Test;
+import software.amazon.s3.analyticsaccelerator.S3SeekableInputStream;
+
+public class TestAnalyticsAcceleratorInputStreamWrapper {
+
+ @Test
+ public void testReadTracksMetrics() throws IOException {
+ S3SeekableInputStream delegate = mock(S3SeekableInputStream.class);
+ // first a single-byte read, then a buffered read of 8 bytes, then a zero-length read, then EOF
+ when(delegate.read()).thenReturn(1);
+ when(delegate.read(any(byte[].class), anyInt(), anyInt()))
+ .thenReturn(8)
+ .thenReturn(0)
+ .thenReturn(-1);
+
+ CachingMetricsContext metrics = new CachingMetricsContext();
+ Counter readBytes = metrics.counter(FileIOMetricsContext.READ_BYTES, MetricsContext.Unit.BYTES);
+ Counter readOperations = metrics.counter(FileIOMetricsContext.READ_OPERATIONS);
+
+ try (AnalyticsAcceleratorInputStreamWrapper in =
+ new AnalyticsAcceleratorInputStreamWrapper(delegate, metrics)) {
+ assertThat(in.read()).isEqualTo(1);
+ assertThat(readBytes.value()).isEqualTo(1);
+ assertThat(readOperations.value()).isEqualTo(1);
+
+ assertThat(in.read(new byte[16], 0, 16)).isEqualTo(8);
+ assertThat(readBytes.value()).isEqualTo(9);
+ assertThat(readOperations.value()).isEqualTo(2);
+
+ // a zero-length read counts neither bytes nor an operation
+ assertThat(in.read(new byte[16], 0, 0)).isEqualTo(0);
+ assertThat(readBytes.value()).isEqualTo(9);
+ assertThat(readOperations.value()).isEqualTo(2);
+
+ // an EOF read counts neither bytes nor an operation
+ assertThat(in.read(new byte[16], 0, 16)).isEqualTo(-1);
+ assertThat(readBytes.value()).isEqualTo(9);
+ assertThat(readOperations.value()).isEqualTo(2);
+ }
+ }
+}
diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3InputStream.java b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3InputStream.java
index d8f415cfd2..5dd6d3dc13 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3InputStream.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3InputStream.java
@@ -18,13 +18,19 @@
*/
package org.apache.iceberg.aws.s3;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
+import org.apache.iceberg.io.FileIOMetricsContext;
+import org.apache.iceberg.metrics.CachingMetricsContext;
+import org.apache.iceberg.metrics.Counter;
+import org.apache.iceberg.metrics.MetricsContext;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -44,13 +50,14 @@ public final class TestS3InputStream {
@BeforeEach
void before() {
- when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class)))
- .thenReturn(inputStream);
s3InputStream = new S3InputStream(s3Client, mock());
}
@Test
void testReadFullyClosesTheStream() throws IOException {
+ when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class)))
+ .thenReturn(inputStream);
+
s3InputStream.readFully(0, new byte[0]);
verify(inputStream).close();
@@ -58,8 +65,89 @@ public final class TestS3InputStream {
@Test
void testReadTailClosesTheStream() throws IOException {
+ when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class)))
+ .thenReturn(inputStream);
+
s3InputStream.readTail(new byte[0], 0, 0);
verify(inputStream).close();
}
+
+ @Test
+ void testReadFullyTracksMetrics() throws IOException {
+ byte[] data = new byte[] {1, 2, 3, 4, 5, 6, 7, 8};
+ when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class)))
+ .thenReturn(new ByteArrayInputStream(data));
+
+ CachingMetricsContext metrics = new CachingMetricsContext();
+ Counter readBytes = metrics.counter(FileIOMetricsContext.READ_BYTES, MetricsContext.Unit.BYTES);
+ Counter readOperations = metrics.counter(FileIOMetricsContext.READ_OPERATIONS);
+
+ try (S3InputStream in =
+ new S3InputStream(s3Client, mock(), new S3FileIOProperties(), metrics)) {
+ in.readFully(0, new byte[data.length], 0, data.length);
+
+ assertThat(readBytes.value()).isEqualTo(data.length);
+ assertThat(readOperations.value()).isEqualTo(1);
+ }
+ }
+
+ @Test
+ void testZeroLengthReadFullyDoesNotCountMetrics() throws IOException {
+ when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class)))
+ .thenReturn(new ByteArrayInputStream(new byte[0]));
+
+ CachingMetricsContext metrics = new CachingMetricsContext();
+ Counter readBytes = metrics.counter(FileIOMetricsContext.READ_BYTES, MetricsContext.Unit.BYTES);
+ Counter readOperations = metrics.counter(FileIOMetricsContext.READ_OPERATIONS);
+
+ try (S3InputStream in =
+ new S3InputStream(s3Client, mock(), new S3FileIOProperties(), metrics)) {
+ in.readFully(0, new byte[0], 0, 0);
+
+ // a zero-length readFully performs no real read; it must count neither bytes nor an operation
+ assertThat(readBytes.value()).isEqualTo(0);
+ assertThat(readOperations.value()).isEqualTo(0);
+ }
+ }
+
+ @Test
+ void testReadTailTracksMetrics() throws IOException {
+ byte[] data = new byte[] {1, 2, 3, 4};
+ when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class)))
+ .thenReturn(new ByteArrayInputStream(data));
+
+ CachingMetricsContext metrics = new CachingMetricsContext();
+ Counter readBytes = metrics.counter(FileIOMetricsContext.READ_BYTES, MetricsContext.Unit.BYTES);
+ Counter readOperations = metrics.counter(FileIOMetricsContext.READ_OPERATIONS);
+
+ try (S3InputStream in =
+ new S3InputStream(s3Client, mock(), new S3FileIOProperties(), metrics)) {
+ int bytesRead = in.readTail(new byte[data.length], 0, data.length);
+
+ assertThat(bytesRead).isEqualTo(data.length);
+ assertThat(readBytes.value()).isEqualTo(data.length);
+ assertThat(readOperations.value()).isEqualTo(1);
+ }
+ }
+
+ @Test
+ void testReadTailEmptyObjectDoesNotCountMetrics() throws IOException {
+ when(s3Client.getObject(any(GetObjectRequest.class), any(ResponseTransformer.class)))
+ .thenReturn(new ByteArrayInputStream(new byte[0]));
+
+ CachingMetricsContext metrics = new CachingMetricsContext();
+ Counter readBytes = metrics.counter(FileIOMetricsContext.READ_BYTES, MetricsContext.Unit.BYTES);
+ Counter readOperations = metrics.counter(FileIOMetricsContext.READ_OPERATIONS);
+
+ try (S3InputStream in =
+ new S3InputStream(s3Client, mock(), new S3FileIOProperties(), metrics)) {
+ int bytesRead = in.readTail(new byte[8], 0, 8);
+
+ // an empty object yields no bytes; a no-data read counts neither bytes nor an operation
+ assertThat(bytesRead).isEqualTo(0);
+ assertThat(readBytes.value()).isEqualTo(0);
+ assertThat(readOperations.value()).isEqualTo(0);
+ }
+ }
}
diff --git a/azure/src/main/java/org/apache/iceberg/azure/adlsv2/ADLSInputStream.java b/azure/src/main/java/org/apache/iceberg/azure/adlsv2/ADLSInputStream.java
index bb96fc8900..4016ca8037 100644
--- a/azure/src/main/java/org/apache/iceberg/azure/adlsv2/ADLSInputStream.java
+++ b/azure/src/main/java/org/apache/iceberg/azure/adlsv2/ADLSInputStream.java
@@ -179,6 +179,10 @@ class ADLSInputStream extends SeekableInputStream implements RangeReadable {
try (InputStream inputStream = openRange(range).getInputStream()) {
IOUtil.readFully(inputStream, buffer, offset, length);
+ if (length > 0) {
+ readBytes.increment(length);
+ readOperations.increment();
+ }
}
}
@@ -189,10 +193,15 @@ class ADLSInputStream extends SeekableInputStream implements RangeReadable {
if (this.fileSize == null) {
this.fileSize = fileClient.getProperties().getFileSize();
}
- long readStart = fileSize - length;
+ long readStart = Math.max(0, fileSize - length);
try (InputStream inputStream = openRange(new FileRange(readStart)).getInputStream()) {
- return IOUtil.readRemaining(inputStream, buffer, offset, length);
+ int bytesRead = IOUtil.readRemaining(inputStream, buffer, offset, length);
+ if (bytesRead > 0) {
+ readBytes.increment(bytesRead);
+ readOperations.increment();
+ }
+ return bytesRead;
}
}
diff --git a/azure/src/test/java/org/apache/iceberg/azure/adlsv2/TestADLSInputStream.java b/azure/src/test/java/org/apache/iceberg/azure/adlsv2/TestADLSInputStream.java
index a104793455..fb8513675a 100644
--- a/azure/src/test/java/org/apache/iceberg/azure/adlsv2/TestADLSInputStream.java
+++ b/azure/src/test/java/org/apache/iceberg/azure/adlsv2/TestADLSInputStream.java
@@ -26,10 +26,14 @@ import static org.mockito.Mockito.when;
import com.azure.storage.file.datalake.DataLakeFileClient;
import com.azure.storage.file.datalake.implementation.models.InternalDataLakeFileOpenInputStreamResult;
+import com.azure.storage.file.datalake.models.PathProperties;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
+import org.apache.iceberg.io.FileIOMetricsContext;
+import org.apache.iceberg.metrics.CachingMetricsContext;
+import org.apache.iceberg.metrics.Counter;
import org.apache.iceberg.metrics.MetricsContext;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -123,4 +127,91 @@ class TestADLSInputStream {
verify(inputStream).close();
}
+
+ @Test
+ void testReadFullyTracksMetrics() throws IOException {
+ byte[] data = new byte[] {1, 2, 3, 4, 5, 6, 7, 8};
+ InputStream byteStream = new ByteArrayInputStream(data);
+ InternalDataLakeFileOpenInputStreamResult openInputStreamResult =
+ new InternalDataLakeFileOpenInputStreamResult(byteStream, mock());
+ when(fileClient.openInputStream(any())).thenReturn(openInputStreamResult);
+
+ CachingMetricsContext metrics = new CachingMetricsContext();
+ Counter readBytes = metrics.counter(FileIOMetricsContext.READ_BYTES, MetricsContext.Unit.BYTES);
+ Counter readOperations = metrics.counter(FileIOMetricsContext.READ_OPERATIONS);
+
+ try (ADLSInputStream in =
+ new ADLSInputStream(
+ "abfs://container@account.dfs.core.windows.net/path/to/file",
+ fileClient,
+ (long) data.length,
+ mock(),
+ metrics)) {
+ in.readFully(0, new byte[data.length], 0, data.length);
+
+ assertThat(readBytes.value()).isEqualTo(data.length);
+ assertThat(readOperations.value()).isEqualTo(1);
+ }
+ }
+
+ @Test
+ void testReadTailTracksMetrics() throws IOException {
+ byte[] data = new byte[] {1, 2, 3, 4, 5, 6, 7, 8};
+ InputStream byteStream = new ByteArrayInputStream(data);
+ // the constructor's openStream() reads the file size from the stream result, so report the
+ // real length; otherwise readTail computes a negative start offset
+ PathProperties properties = mock(PathProperties.class);
+ when(properties.getFileSize()).thenReturn((long) data.length);
+ InternalDataLakeFileOpenInputStreamResult openInputStreamResult =
+ new InternalDataLakeFileOpenInputStreamResult(byteStream, properties);
+ when(fileClient.openInputStream(any())).thenReturn(openInputStreamResult);
+
+ CachingMetricsContext metrics = new CachingMetricsContext();
+ Counter readBytes = metrics.counter(FileIOMetricsContext.READ_BYTES, MetricsContext.Unit.BYTES);
+ Counter readOperations = metrics.counter(FileIOMetricsContext.READ_OPERATIONS);
+
+ try (ADLSInputStream in =
+ new ADLSInputStream(
+ "abfs://container@account.dfs.core.windows.net/path/to/file",
+ fileClient,
+ (long) data.length,
+ mock(),
+ metrics)) {
+ int tailLength = 4;
+ int bytesRead = in.readTail(new byte[tailLength], 0, tailLength);
+
+ assertThat(bytesRead).isEqualTo(tailLength);
+ assertThat(readBytes.value()).isEqualTo(tailLength);
+ assertThat(readOperations.value()).isEqualTo(1);
+ }
+ }
+
+ @Test
+ void testReadTailEmptyObjectDoesNotCountMetrics() throws IOException {
+ InputStream byteStream = new ByteArrayInputStream(new byte[0]);
+ PathProperties properties = mock(PathProperties.class);
+ when(properties.getFileSize()).thenReturn(0L);
+ InternalDataLakeFileOpenInputStreamResult openInputStreamResult =
+ new InternalDataLakeFileOpenInputStreamResult(byteStream, properties);
+ when(fileClient.openInputStream(any())).thenReturn(openInputStreamResult);
+
+ CachingMetricsContext metrics = new CachingMetricsContext();
+ Counter readBytes = metrics.counter(FileIOMetricsContext.READ_BYTES, MetricsContext.Unit.BYTES);
+ Counter readOperations = metrics.counter(FileIOMetricsContext.READ_OPERATIONS);
+
+ try (ADLSInputStream in =
+ new ADLSInputStream(
+ "abfs://container@account.dfs.core.windows.net/path/to/file",
+ fileClient,
+ 0L,
+ mock(),
+ metrics)) {
+ int bytesRead = in.readTail(new byte[8], 0, 8);
+
+ // an empty object yields no bytes; a no-data read counts neither bytes nor an operation
+ assertThat(bytesRead).isEqualTo(0);
+ assertThat(readBytes.value()).isEqualTo(0);
+ assertThat(readOperations.value()).isEqualTo(0);
+ }
+ }
}
diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/gcs/AnalyticsCoreUtil.java b/gcp/src/main/java/org/apache/iceberg/gcp/gcs/AnalyticsCoreUtil.java
index 4bc9659d77..31521ce256 100644
--- a/gcp/src/main/java/org/apache/iceberg/gcp/gcs/AnalyticsCoreUtil.java
+++ b/gcp/src/main/java/org/apache/iceberg/gcp/gcs/AnalyticsCoreUtil.java
@@ -141,8 +141,10 @@ class AnalyticsCoreUtil {
GCSExceptionUtil.throwNotFoundIfNotPresent(e, blobId);
throw e;
}
- readBytes.increment();
- readOperations.increment();
+ if (readByte != -1) {
+ readBytes.increment();
+ readOperations.increment();
+ }
return readByte;
}
@@ -162,8 +164,8 @@ class AnalyticsCoreUtil {
}
if (bytesRead > 0) {
readBytes.increment(bytesRead);
+ readOperations.increment();
}
- readOperations.increment();
return bytesRead;
}
@@ -175,16 +177,26 @@ class AnalyticsCoreUtil {
GCSExceptionUtil.throwNotFoundIfNotPresent(e, blobId);
throw e;
}
+ if (length > 0) {
+ readBytes.increment(length);
+ readOperations.increment();
+ }
}
@Override
public int readTail(byte[] buffer, int offset, int length) throws IOException {
+ int bytesRead;
try {
- return stream.readTail(buffer, offset, length);
+ bytesRead = stream.readTail(buffer, offset, length);
} catch (IOException e) {
GCSExceptionUtil.throwNotFoundIfNotPresent(e, blobId);
throw e;
}
+ if (bytesRead > 0) {
+ readBytes.increment(bytesRead);
+ readOperations.increment();
+ }
+ return bytesRead;
}
@Override
@@ -200,6 +212,22 @@ class AnalyticsCoreUtil {
.setByteBufferFuture(fileRange.byteBuffer())
.build())
.collect(Collectors.toList());
+ // Count synchronously on the caller (task) thread, before delegating. The range futures
+ // complete on analytics-core background threads, so counting in their completion callbacks
+ // would attribute the bytes to the wrong thread: under HadoopMetricsContext, READ_BYTES maps
+ // to FileSystem.Statistics.incrementBytesRead, which accumulates per-thread, and Spark reads
+ // task input bytes from the task thread's statistics. Bytes recorded on a background thread
+ // would never reach Spark's task metrics. Correct-thread attribution matters more than
+ // precision here, so we count the requested range.length() up front. This over-counts by a
+ // bounded amount on a short read near EOF, and counts ranges whose read later fails (the
+ // delegate throws below, but the bytes were already recorded); both are acceptable for a
+ // metric where the magnitude is right and it lands where Spark can see it.
+ for (FileRange range : ranges) {
+ if (range.length() > 0) {
+ readBytes.increment(range.length());
+ readOperations.increment();
+ }
+ }
try {
stream.readVectored(objectRanges, allocate);
} catch (IOException e) {
diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/gcs/GCSInputStream.java b/gcp/src/main/java/org/apache/iceberg/gcp/gcs/GCSInputStream.java
index 1f59b8ffa2..31199d8521 100644
--- a/gcp/src/main/java/org/apache/iceberg/gcp/gcs/GCSInputStream.java
+++ b/gcp/src/main/java/org/apache/iceberg/gcp/gcs/GCSInputStream.java
@@ -168,6 +168,10 @@ class GCSInputStream extends SeekableInputStream implements RangeReadable {
throw new EOFException(
"Reached the end of stream with " + (length - bytesRead) + " bytes left to read");
}
+ if (bytesRead > 0) {
+ readBytes.increment(bytesRead);
+ readOperations.increment();
+ }
}
}
@@ -179,7 +183,12 @@ class GCSInputStream extends SeekableInputStream implements RangeReadable {
long startPosition = Math.max(0, blobSize - length);
try (ReadChannel readChannel = openChannel()) {
readChannel.seek(startPosition);
- return read(readChannel, ByteBuffer.wrap(buffer), offset, length);
+ int bytesRead = read(readChannel, ByteBuffer.wrap(buffer), offset, length);
+ if (bytesRead > 0) {
+ readBytes.increment(bytesRead);
+ readOperations.increment();
+ }
+ return bytesRead;
}
}
diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/gcs/TestAnalyticsCoreUtil.java b/gcp/src/test/java/org/apache/iceberg/gcp/gcs/TestAnalyticsCoreUtil.java
index d915553453..63a8a43285 100644
--- a/gcp/src/test/java/org/apache/iceberg/gcp/gcs/TestAnalyticsCoreUtil.java
+++ b/gcp/src/test/java/org/apache/iceberg/gcp/gcs/TestAnalyticsCoreUtil.java
@@ -18,11 +18,14 @@
*/
package org.apache.iceberg.gcp.gcs;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import com.google.cloud.gcs.analyticscore.client.GcsFileSystem;
import com.google.cloud.gcs.analyticscore.client.GcsItemId;
@@ -34,9 +37,12 @@ import java.nio.ByteBuffer;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.function.IntFunction;
+import org.apache.iceberg.io.FileIOMetricsContext;
import org.apache.iceberg.io.FileRange;
import org.apache.iceberg.io.RangeReadable;
import org.apache.iceberg.io.SeekableInputStream;
+import org.apache.iceberg.metrics.CachingMetricsContext;
+import org.apache.iceberg.metrics.Counter;
import org.apache.iceberg.metrics.MetricsContext;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
@@ -80,4 +86,150 @@ public class TestAnalyticsCoreUtil {
.build());
verify(gcsInputStream).readVectored(objectRanges, allocate);
}
+
+ @Test
+ public void readVectoredCountsRequestedLengthSynchronously() throws IOException {
+ GcsFileSystem fileSystem = mock(GcsFileSystem.class);
+ GoogleCloudStorageInputStream gcsInputStream = mock(GoogleCloudStorageInputStream.class);
+ BlobId blobId = BlobId.of("mockbucket", "mockname");
+
+ CachingMetricsContext metrics = new CachingMetricsContext();
+ Counter readBytes = metrics.counter(FileIOMetricsContext.READ_BYTES, MetricsContext.Unit.BYTES);
+ Counter readOperations = metrics.counter(FileIOMetricsContext.READ_OPERATIONS);
+
+ SeekableInputStream stream;
+ try (MockedStatic<GoogleCloudStorageInputStream> mocked =
+ mockStatic(GoogleCloudStorageInputStream.class)) {
+ mocked
+ .when(() -> GoogleCloudStorageInputStream.create(eq(fileSystem), any(GcsItemId.class)))
+ .thenReturn(gcsInputStream);
+ stream = AnalyticsCoreUtil.newStream(fileSystem, blobId, null, metrics);
+ }
+
+ CompletableFuture<ByteBuffer> future1 = new CompletableFuture<>();
+ CompletableFuture<ByteBuffer> future2 = new CompletableFuture<>();
+ List<FileRange> ranges =
+ List.of(new FileRange(future1, 10L, 100), new FileRange(future2, 200L, 50));
+
+ ((RangeReadable) stream).readVectored(ranges, ByteBuffer::allocate);
+
+ // Bytes are counted synchronously on the caller thread, using the requested range length,
+ // before the range futures complete. This is deliberate: the futures complete on analytics-core
+ // background threads, and counting there would attribute bytes to the wrong thread's Hadoop
+ // Statistics and never reach Spark's task input metrics (see AnalyticsCoreUtil#readVectored).
+ // The tradeoff is a bounded over-count on short reads / failed ranges.
+ //
+ // NOTE: this test uses a plain LongAdder-backed counter, which cannot verify the per-thread
+ // Statistics attribution that motivates counting synchronously; that correctness argument
+ // rests on the HadoopMetricsContext reasoning, not on this assertion.
+ assertThat(readBytes.value()).isEqualTo(150);
+ assertThat(readOperations.value()).isEqualTo(2);
+
+ // completing (or failing) the futures afterward does not change the already-recorded metrics
+ future1.complete(ByteBuffer.allocate(100));
+ future2.completeExceptionally(new IOException("boom"));
+ assertThat(readBytes.value()).isEqualTo(150);
+ assertThat(readOperations.value()).isEqualTo(2);
+ }
+
+ @Test
+ public void readDoesNotCountAtEof() throws IOException {
+ GcsFileSystem fileSystem = mock(GcsFileSystem.class);
+ GoogleCloudStorageInputStream gcsInputStream = mock(GoogleCloudStorageInputStream.class);
+ BlobId blobId = BlobId.of("mockbucket", "mockname");
+ // a real byte, then EOF; a buffered read of 8 bytes, then EOF
+ when(gcsInputStream.read()).thenReturn(42).thenReturn(-1);
+ when(gcsInputStream.read(any(byte[].class), anyInt(), anyInt())).thenReturn(8).thenReturn(-1);
+
+ CachingMetricsContext metrics = new CachingMetricsContext();
+ Counter readBytes = metrics.counter(FileIOMetricsContext.READ_BYTES, MetricsContext.Unit.BYTES);
+ Counter readOperations = metrics.counter(FileIOMetricsContext.READ_OPERATIONS);
+
+ SeekableInputStream stream;
+ try (MockedStatic<GoogleCloudStorageInputStream> mocked =
+ mockStatic(GoogleCloudStorageInputStream.class)) {
+ mocked
+ .when(() -> GoogleCloudStorageInputStream.create(eq(fileSystem), any(GcsItemId.class)))
+ .thenReturn(gcsInputStream);
+ stream = AnalyticsCoreUtil.newStream(fileSystem, blobId, null, metrics);
+ }
+
+ assertThat(stream.read()).isEqualTo(42);
+ assertThat(readBytes.value()).isEqualTo(1);
+ assertThat(readOperations.value()).isEqualTo(1);
+
+ assertThat(stream.read(new byte[16], 0, 16)).isEqualTo(8);
+ assertThat(readBytes.value()).isEqualTo(9);
+ assertThat(readOperations.value()).isEqualTo(2);
+
+ // EOF reads count neither bytes nor an operation
+ assertThat(stream.read()).isEqualTo(-1);
+ assertThat(stream.read(new byte[16], 0, 16)).isEqualTo(-1);
+ assertThat(readBytes.value()).isEqualTo(9);
+ assertThat(readOperations.value()).isEqualTo(2);
+ }
+
+ @Test
+ public void readVectoredSkipsZeroLengthRanges() throws IOException {
+ GcsFileSystem fileSystem = mock(GcsFileSystem.class);
+ GoogleCloudStorageInputStream gcsInputStream = mock(GoogleCloudStorageInputStream.class);
+ BlobId blobId = BlobId.of("mockbucket", "mockname");
+
+ CachingMetricsContext metrics = new CachingMetricsContext();
+ Counter readBytes = metrics.counter(FileIOMetricsContext.READ_BYTES, MetricsContext.Unit.BYTES);
+ Counter readOperations = metrics.counter(FileIOMetricsContext.READ_OPERATIONS);
+
+ SeekableInputStream stream;
+ try (MockedStatic<GoogleCloudStorageInputStream> mocked =
+ mockStatic(GoogleCloudStorageInputStream.class)) {
+ mocked
+ .when(() -> GoogleCloudStorageInputStream.create(eq(fileSystem), any(GcsItemId.class)))
+ .thenReturn(gcsInputStream);
+ stream = AnalyticsCoreUtil.newStream(fileSystem, blobId, null, metrics);
+ }
+
+ CompletableFuture<ByteBuffer> future1 = new CompletableFuture<>();
+ CompletableFuture<ByteBuffer> future2 = new CompletableFuture<>();
+ // a zero-length range must not count bytes or an operation
+ List<FileRange> ranges =
+ List.of(new FileRange(future1, 10L, 100), new FileRange(future2, 200L, 0));
+
+ ((RangeReadable) stream).readVectored(ranges, ByteBuffer::allocate);
+
+ assertThat(readBytes.value()).isEqualTo(100);
+ assertThat(readOperations.value()).isEqualTo(1);
+ }
+
+ @Test
+ public void readTailDoesNotCountAtEof() throws IOException {
+ GcsFileSystem fileSystem = mock(GcsFileSystem.class);
+ GoogleCloudStorageInputStream gcsInputStream = mock(GoogleCloudStorageInputStream.class);
+ BlobId blobId = BlobId.of("mockbucket", "mockname");
+ // a tail read of 8 bytes, then an empty/EOF tail read
+ when(gcsInputStream.readTail(any(byte[].class), anyInt(), anyInt()))
+ .thenReturn(8)
+ .thenReturn(-1);
+
+ CachingMetricsContext metrics = new CachingMetricsContext();
+ Counter readBytes = metrics.counter(FileIOMetricsContext.READ_BYTES, MetricsContext.Unit.BYTES);
+ Counter readOperations = metrics.counter(FileIOMetricsContext.READ_OPERATIONS);
+
+ SeekableInputStream stream;
+ try (MockedStatic<GoogleCloudStorageInputStream> mocked =
+ mockStatic(GoogleCloudStorageInputStream.class)) {
+ mocked
+ .when(() -> GoogleCloudStorageInputStream.create(eq(fileSystem), any(GcsItemId.class)))
+ .thenReturn(gcsInputStream);
+ stream = AnalyticsCoreUtil.newStream(fileSystem, blobId, null, metrics);
+ }
+
+ assertThat(((RangeReadable) stream).readTail(new byte[16], 0, 16)).isEqualTo(8);
+ assertThat(readBytes.value()).isEqualTo(8);
+ assertThat(readOperations.value()).isEqualTo(1);
+
+ // an empty/EOF tail read counts neither bytes nor an operation
+ assertThat(((RangeReadable) stream).readTail(new byte[16], 0, 16)).isEqualTo(-1);
+ assertThat(readBytes.value()).isEqualTo(8);
+ assertThat(readOperations.value()).isEqualTo(1);
+ }
}
diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/gcs/TestGCSInputStream.java b/gcp/src/test/java/org/apache/iceberg/gcp/gcs/TestGCSInputStream.java
index 107b668fa9..b909396934 100644
--- a/gcp/src/test/java/org/apache/iceberg/gcp/gcs/TestGCSInputStream.java
+++ b/gcp/src/test/java/org/apache/iceberg/gcp/gcs/TestGCSInputStream.java
@@ -30,9 +30,12 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Random;
import org.apache.iceberg.gcp.GCPProperties;
+import org.apache.iceberg.io.FileIOMetricsContext;
import org.apache.iceberg.io.IOUtil;
import org.apache.iceberg.io.RangeReadable;
import org.apache.iceberg.io.SeekableInputStream;
+import org.apache.iceberg.metrics.CachingMetricsContext;
+import org.apache.iceberg.metrics.Counter;
import org.apache.iceberg.metrics.MetricsContext;
import org.junit.jupiter.api.Test;
@@ -181,6 +184,49 @@ public class TestGCSInputStream {
.isEqualTo(Arrays.copyOfRange(original, offset, offset + length));
}
+ @Test
+ public void testRangeReadMetrics() throws Exception {
+ BlobId uri = BlobId.fromGsUtilUri("gs://bucket/path/to/read-metrics.dat");
+ int dataSize = 1024 * 1024;
+ byte[] data = randomData(dataSize);
+ writeGCSData(uri, data);
+
+ CachingMetricsContext metrics = new CachingMetricsContext();
+ Counter readBytes = metrics.counter(FileIOMetricsContext.READ_BYTES, MetricsContext.Unit.BYTES);
+ Counter readOperations = metrics.counter(FileIOMetricsContext.READ_OPERATIONS);
+
+ try (RangeReadable in = new GCSInputStream(storage, uri, null, gcpProperties, metrics)) {
+ int length = 1024;
+ in.readFully(0, new byte[length], 0, length);
+ assertThat(readBytes.value()).isEqualTo(length);
+ assertThat(readOperations.value()).isEqualTo(1);
+
+ int tailLength = 512;
+ in.readTail(new byte[tailLength], 0, tailLength);
+ assertThat(readBytes.value()).isEqualTo(length + tailLength);
+ assertThat(readOperations.value()).isEqualTo(2);
+ }
+ }
+
+ @Test
+ public void testReadTailEmptyObjectDoesNotDecrementMetrics() throws Exception {
+ BlobId uri = BlobId.fromGsUtilUri("gs://bucket/path/to/empty.dat");
+ writeGCSData(uri, new byte[0]);
+
+ CachingMetricsContext metrics = new CachingMetricsContext();
+ Counter readBytes = metrics.counter(FileIOMetricsContext.READ_BYTES, MetricsContext.Unit.BYTES);
+ Counter readOperations = metrics.counter(FileIOMetricsContext.READ_OPERATIONS);
+
+ try (RangeReadable in = new GCSInputStream(storage, uri, 0L, gcpProperties, metrics)) {
+ int bytesRead = in.readTail(new byte[8], 0, 8);
+
+ // an empty object yields no bytes; a no-data read counts neither bytes nor an operation
+ assertThat(bytesRead).isLessThanOrEqualTo(0);
+ assertThat(readBytes.value()).isEqualTo(0);
+ assertThat(readOperations.value()).isEqualTo(0);
+ }
+ }
+
@Test
public void testClose() throws Exception {
BlobId blobId = BlobId.fromGsUtilUri("gs://bucket/path/to/closed.dat");