You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by ya...@apache.org on 2026/09/17 05:53:24 UTC

(fluss) branch main updated: [metrics] Add KV snapshot upload throughput metrics

This is an automated email from the ASF dual-hosted git repository.

platinumhamburg pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git


The following commit(s) were added to refs/heads/main by this push:
     new aade48b0d [metrics] Add KV snapshot upload throughput metrics
aade48b0d is described below

commit aade48b0dd6f968015d2c3e183242d513c1127be
Author: chaohaochen <57...@users.noreply.github.com>
AuthorDate: Thu Sep 17 13:53:20 2026 +0800

    [metrics] Add KV snapshot upload throughput metrics
    
    Co-authored-by: 韵成 <ch...@alibaba-inc.com>
---
 .../java/org/apache/fluss/metrics/MetricNames.java |   2 +
 .../java/org/apache/fluss/server/kv/KvTablet.java  |   8 +-
 .../server/kv/snapshot/KvSnapshotDataUploader.java |  23 ++-
 .../kv/snapshot/RocksIncrementalSnapshot.java      |  14 +-
 .../server/metrics/group/TableMetricGroup.java     |  15 ++
 .../org/apache/fluss/server/replica/Replica.java   |   3 +-
 .../kv/snapshot/KvSnapshotDataUploaderTest.java    | 199 ++++++++++++++++++++-
 .../kv/snapshot/KvTabletSnapshotTargetTest.java    |  24 ++-
 .../kv/snapshot/RocksIncrementalSnapshotTest.java  |  41 ++++-
 .../server/metrics/group/TableMetricGroupTest.java |  75 ++++++++
 .../apache/fluss/server/replica/ReplicaTest.java   |  10 ++
 .../maintenance/observability/monitor-metrics.md   |   9 +-
 12 files changed, 389 insertions(+), 34 deletions(-)

diff --git a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java
index ac60ee5f1..0219f3fe7 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java
@@ -135,6 +135,8 @@ public class MetricNames {
     public static final String FAILED_PRODUCE_FETCH_LOG_REQUESTS_RATE =
             "failedProduceLogRequestsPerSecond";
 
+    public static final String REMOTE_KV_COPY_BYTES_RATE = "remoteKvCopyBytesPerSecond";
+
     public static final String REMOTE_LOG_COPY_BYTES_RATE = "remoteLogCopyBytesPerSecond";
     public static final String REMOTE_LOG_COPY_REQUESTS_RATE = "remoteLogCopyRequestsPerSecond";
     public static final String REMOTE_LOG_COPY_ERROR_RATE = "remoteLogCopyErrorPerSecond";
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
index 2312b6fd4..61c5a2d5d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
@@ -32,6 +32,7 @@ import org.apache.fluss.metadata.PhysicalTablePath;
 import org.apache.fluss.metadata.SchemaGetter;
 import org.apache.fluss.metadata.TableBucket;
 import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.metrics.Counter;
 import org.apache.fluss.record.ChangeType;
 import org.apache.fluss.record.KvRecordBatch;
 import org.apache.fluss.row.arrow.ArrowWriterPool;
@@ -1369,17 +1370,20 @@ public final class KvTablet {
                 });
     }
 
+    /** Creates an incremental snapshot with the table's thread-safe upload byte counter. */
     public RocksIncrementalSnapshot createIncrementalSnapshot(
             Map<Long, Collection<KvFileHandleAndLocalPath>> uploadedSstFiles,
             KvSnapshotDataUploader kvSnapshotDataUploader,
-            long lastCompletedSnapshotId) {
+            long lastCompletedSnapshotId,
+            Counter remoteKvCopyBytes) {
         return new RocksIncrementalSnapshot(
                 uploadedSstFiles,
                 rocksDBKv.getDb(),
                 rocksDBKv.getResourceGuard(),
                 kvSnapshotDataUploader,
                 kvTabletDir,
-                lastCompletedSnapshotId);
+                lastCompletedSnapshotId,
+                remoteKvCopyBytes);
     }
 
     // only for testing.
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/KvSnapshotDataUploader.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/KvSnapshotDataUploader.java
index ef4d3b6ee..e0f958ec6 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/KvSnapshotDataUploader.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/KvSnapshotDataUploader.java
@@ -18,6 +18,7 @@
 package org.apache.fluss.server.kv.snapshot;
 
 import org.apache.fluss.exception.FlussRuntimeException;
+import org.apache.fluss.metrics.Counter;
 import org.apache.fluss.server.utils.SnapshotUtil;
 import org.apache.fluss.utils.CloseableRegistry;
 import org.apache.fluss.utils.ExceptionUtils;
@@ -51,6 +52,7 @@ public class KvSnapshotDataUploader extends KvSnapshotDataTransfer {
      * Upload all the files to the target snapshot location using specified number of threads.
      *
      * @param files The files will be uploaded to the snapshot location.
+     * @param remoteKvCopyBytes Thread-safe counter for successfully uploaded bytes of this table.
      * @throws Exception Thrown if can not upload all the files.
      */
     public List<KvFileHandleAndLocalPath> uploadFilesToSnapshotLocation(
@@ -58,7 +60,8 @@ public class KvSnapshotDataUploader extends KvSnapshotDataTransfer {
             SnapshotLocation snapshotLocation,
             SnapshotFileScope snapshotFileScope,
             CloseableRegistry closeableRegistry,
-            CloseableRegistry tmpResourcesRegistry)
+            CloseableRegistry tmpResourcesRegistry,
+            Counter remoteKvCopyBytes)
             throws Exception {
 
         List<CompletableFuture<KvFileHandleAndLocalPath>> futures =
@@ -67,7 +70,8 @@ public class KvSnapshotDataUploader extends KvSnapshotDataTransfer {
                         snapshotLocation,
                         snapshotFileScope,
                         closeableRegistry,
-                        tmpResourcesRegistry);
+                        tmpResourcesRegistry,
+                        remoteKvCopyBytes);
 
         List<KvFileHandleAndLocalPath> handles = new ArrayList<>(files.size());
 
@@ -95,7 +99,8 @@ public class KvSnapshotDataUploader extends KvSnapshotDataTransfer {
             SnapshotLocation snapshotLocation,
             SnapshotFileScope snapshotFileScope,
             CloseableRegistry closeableRegistry,
-            CloseableRegistry tmpResourcesRegistry) {
+            CloseableRegistry tmpResourcesRegistry,
+            Counter remoteKvCopyBytes) {
         return files.stream()
                 .map(
                         e ->
@@ -107,7 +112,8 @@ public class KvSnapshotDataUploader extends KvSnapshotDataTransfer {
                                                                 snapshotLocation,
                                                                 snapshotFileScope,
                                                                 closeableRegistry,
-                                                                tmpResourcesRegistry)),
+                                                                tmpResourcesRegistry,
+                                                                remoteKvCopyBytes)),
                                         dataTransferThreadPool))
                 .collect(Collectors.toList());
     }
@@ -117,7 +123,8 @@ public class KvSnapshotDataUploader extends KvSnapshotDataTransfer {
             SnapshotLocation snapshotLocation,
             SnapshotFileScope snapshotFileScope,
             CloseableRegistry closeableRegistry,
-            CloseableRegistry tmpResourcesRegistry)
+            CloseableRegistry tmpResourcesRegistry,
+            Counter remoteKvCopyBytes)
             throws IOException {
 
         InputStream inputStream = null;
@@ -132,6 +139,7 @@ public class KvSnapshotDataUploader extends KvSnapshotDataTransfer {
             outputStream = snapshotLocation.createSnapshotOutputStream(snapshotFileScope);
             closeableRegistry.registerCloseable(outputStream);
 
+            long uploadedBytes = 0L;
             while (true) {
                 int numBytes = inputStream.read(buffer);
 
@@ -139,11 +147,16 @@ public class KvSnapshotDataUploader extends KvSnapshotDataTransfer {
                     break;
                 }
                 outputStream.write(buffer, 0, numBytes);
+                uploadedBytes += numBytes;
             }
 
             final KvFileHandle result;
             if (closeableRegistry.unregisterCloseable(outputStream)) {
                 result = outputStream.closeAndGetHandle();
+                if (result != null) {
+                    // Count the uploaded bytes even if this snapshot later fails or aborts.
+                    remoteKvCopyBytes.inc(uploadedBytes);
+                }
             } else {
                 result = null;
             }
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/RocksIncrementalSnapshot.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/RocksIncrementalSnapshot.java
index c50fe3ef0..9cde2d5d8 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/RocksIncrementalSnapshot.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/RocksIncrementalSnapshot.java
@@ -17,6 +17,7 @@
 
 package org.apache.fluss.server.kv.snapshot;
 
+import org.apache.fluss.metrics.Counter;
 import org.apache.fluss.server.utils.ResourceGuard;
 import org.apache.fluss.utils.CloseableRegistry;
 import org.apache.fluss.utils.ExceptionUtils;
@@ -80,13 +81,18 @@ public class RocksIncrementalSnapshot implements AutoCloseable {
     /** The help class used to upload kv snapshot files. */
     private final KvSnapshotDataUploader kvSnapshotDataUploader;
 
+    private final Counter remoteKvCopyBytes;
+
+    /** Creates an incremental snapshot using the table's thread-safe upload byte counter. */
     public RocksIncrementalSnapshot(
             Map<Long, Collection<KvFileHandleAndLocalPath>> uploadedSstFiles,
             @Nonnull RocksDB db,
             ResourceGuard rocksDBResourceGuard,
             KvSnapshotDataUploader kvSnapshotDataUploader,
             @Nonnull File instanceBasePath,
-            long lastCompletedSnapshotId) {
+            long lastCompletedSnapshotId,
+            Counter remoteKvCopyBytes) {
+        this.remoteKvCopyBytes = remoteKvCopyBytes;
         this.uploadedSstFiles = uploadedSstFiles;
         this.db = db;
         this.rocksDBResourceGuard = rocksDBResourceGuard;
@@ -253,7 +259,8 @@ public class RocksIncrementalSnapshot implements AutoCloseable {
                             snapshotLocation,
                             SnapshotFileScope.SHARED,
                             closeableRegistry,
-                            tmpResourcesRegistry);
+                            tmpResourcesRegistry,
+                            remoteKvCopyBytes);
             size +=
                     sstFilesUploadResult.stream()
                             .mapToLong(e -> e.getKvFileHandle().getSize())
@@ -268,7 +275,8 @@ public class RocksIncrementalSnapshot implements AutoCloseable {
                             snapshotLocation,
                             SnapshotFileScope.EXCLUSIVE,
                             closeableRegistry,
-                            tmpResourcesRegistry);
+                            tmpResourcesRegistry,
+                            remoteKvCopyBytes);
             size +=
                     miscFilesUploadResult.stream()
                             .mapToLong(e -> e.getKvFileHandle().getSize())
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java
index 02be54ed9..523045872 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java
@@ -169,6 +169,15 @@ public class TableMetricGroup extends AbstractMetricGroup {
         return logMetrics.remoteLogDeleteErrors;
     }
 
+    /** Returns the successful KV snapshot upload byte counter, or a no-op for log tables. */
+    public Counter remoteKvCopyBytes() {
+        if (kvMetrics == null) {
+            return NoOpCounter.INSTANCE;
+        } else {
+            return kvMetrics.remoteKvCopyBytes;
+        }
+    }
+
     public void incKvMessageIn(long n) {
         if (kvMetrics == null) {
             NoOpCounter.INSTANCE.inc(n);
@@ -592,6 +601,8 @@ public class TableMetricGroup extends AbstractMetricGroup {
 
         private static final String LOOKUP_FILE_DOWNLOADED = "lookup_file_downloaded";
 
+        private final Counter remoteKvCopyBytes;
+
         private final Counter totalLookupRequests;
         private final Counter failedLookupRequests;
         private final Counter totalHistoricalLookupRequests;
@@ -610,6 +621,10 @@ public class TableMetricGroup extends AbstractMetricGroup {
         public KvMetricGroup(TableMetricGroup tableMetricGroup) {
             super(tableMetricGroup, TabletType.KV);
 
+            // for kv snapshot upload
+            remoteKvCopyBytes = new ThreadSafeSimpleCounter();
+            meter(MetricNames.REMOTE_KV_COPY_BYTES_RATE, new MeterView(remoteKvCopyBytes));
+
             // for lookup request
             totalLookupRequests = new ThreadSafeSimpleCounter();
             meter(MetricNames.TOTAL_LOOKUP_REQUESTS_RATE, new MeterView(totalLookupRequests));
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
index a74056ebc..1c33025fc 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
@@ -1131,7 +1131,8 @@ public final class Replica {
                     kvTablet.createIncrementalSnapshot(
                             uploadedSstFiles,
                             snapshotContext.getSnapshotDataUploader(),
-                            lastCompletedSnapshotId);
+                            lastCompletedSnapshotId,
+                            tableMetrics().remoteKvCopyBytes());
 
             // create snapshot ID counter
             SequenceIDCounter snapshotIDCounter =
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvSnapshotDataUploaderTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvSnapshotDataUploaderTest.java
index ef5048593..9432b9abc 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvSnapshotDataUploaderTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvSnapshotDataUploaderTest.java
@@ -20,6 +20,8 @@ package org.apache.fluss.server.kv.snapshot;
 import org.apache.fluss.fs.FSDataInputStream;
 import org.apache.fluss.fs.FsPath;
 import org.apache.fluss.fs.local.LocalFileSystem;
+import org.apache.fluss.metrics.Counter;
+import org.apache.fluss.metrics.ThreadSafeSimpleCounter;
 import org.apache.fluss.utils.CloseableRegistry;
 import org.apache.fluss.utils.IOUtils;
 
@@ -27,39 +29,52 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.ValueSource;
 
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadLocalRandom;
 
+import static org.apache.fluss.testutils.common.CommonTestUtils.retry;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 /** Test class for {@link org.apache.fluss.server.kv.snapshot.KvSnapshotDataUploader}. */
 class KvSnapshotDataUploaderTest {
 
     @TempDir private Path temporaryFolder;
-    private ExecutorService downLoaderThreadPool;
+    private ExecutorService uploaderThreadPool;
 
     @BeforeEach
     void beforeEach() {
-        downLoaderThreadPool = Executors.newSingleThreadExecutor();
+        uploaderThreadPool = Executors.newFixedThreadPool(4);
     }
 
     @AfterEach
     void afterEach() {
-        if (downLoaderThreadPool != null) {
-            downLoaderThreadPool.shutdownNow();
+        if (uploaderThreadPool != null) {
+            uploaderThreadPool.shutdownNow();
         }
     }
 
-    /** Test that the exception arose in the thread pool will rethrow to the main thread. */
+    /** Verifies file contents and byte accounting across concurrent uploads. */
     @Test
     void testMultiThreadUploadCorrectly() throws Exception {
         File snapshotSharedFolder = new File(temporaryFolder.toFile(), "shared");
@@ -80,16 +95,20 @@ class KvSnapshotDataUploaderTest {
         List<Path> sstFilePaths =
                 generateRandomSstFiles(localFolder, sstFileCount, fileSizeThreshold);
 
-        KvSnapshotDataUploader snapshotUploader = new KvSnapshotDataUploader(downLoaderThreadPool);
+        KvSnapshotDataUploader snapshotUploader = new KvSnapshotDataUploader(uploaderThreadPool);
+        Counter counter = new ThreadSafeSimpleCounter();
         List<KvFileHandleAndLocalPath> sstFiles =
                 snapshotUploader.uploadFilesToSnapshotLocation(
                         sstFilePaths,
                         snapshotLocation,
                         SnapshotFileScope.SHARED,
                         new CloseableRegistry(),
-                        new CloseableRegistry());
+                        new CloseableRegistry(),
+                        counter);
 
+        long expectedBytes = 0L;
         for (Path path : sstFilePaths) {
+            expectedBytes += Files.size(path);
             KvFileHandle kvFileHandle =
                     sstFiles.stream()
                             .filter(e -> e.getLocalPath().equals(path.getFileName().toString()))
@@ -99,11 +118,173 @@ class KvSnapshotDataUploaderTest {
             assertThat(kvFileHandle.getFilePath())
                     .startsWith(LocalFileSystem.getLocalFsURI().getScheme());
             FsPath fsPath = new FsPath(kvFileHandle.getFilePath());
-            FSDataInputStream inputStream = fsPath.getFileSystem().open(fsPath);
-            assertContentEqual(path, inputStream);
+            try (FSDataInputStream inputStream = fsPath.getFileSystem().open(fsPath)) {
+                assertContentEqual(path, inputStream);
+            }
+        }
+        assertThat(counter.getCount()).isEqualTo(expectedBytes);
+    }
+
+    @ParameterizedTest
+    @EnumSource(SnapshotFileScope.class)
+    void testSuccessfulUploadsAndReuploads(SnapshotFileScope scope) throws Exception {
+        Path file = Files.write(temporaryFolder.resolve("snapshot-file"), new byte[123]);
+        Counter counter = new ThreadSafeSimpleCounter();
+        Counter otherTableCounter = new ThreadSafeSimpleCounter();
+        KvSnapshotDataUploader uploader = new KvSnapshotDataUploader(uploaderThreadPool);
+        try (CloseableRegistry registry = new CloseableRegistry();
+                CloseableRegistry resources = new CloseableRegistry()) {
+            for (int i = 1; i <= 2; i++) {
+                uploader.uploadFilesToSnapshotLocation(
+                        Collections.singletonList(file),
+                        createSnapshotLocation(),
+                        scope,
+                        registry,
+                        resources,
+                        counter);
+                assertThat(counter.getCount()).isEqualTo(123L * i);
+            }
+            uploader.uploadFilesToSnapshotLocation(
+                    Collections.singletonList(file),
+                    createSnapshotLocation(),
+                    scope,
+                    registry,
+                    resources,
+                    otherTableCounter);
+            assertThat(otherTableCounter.getCount()).isEqualTo(123L);
+            assertThat(counter.getCount()).isEqualTo(246L);
+            uploader.uploadFilesToSnapshotLocation(
+                    Collections.emptyList(),
+                    createSnapshotLocation(),
+                    scope,
+                    registry,
+                    resources,
+                    counter);
+            assertThat(counter.getCount()).isEqualTo(246L);
+        }
+        // Cleanup of an aborted snapshot must not roll back successful upload traffic.
+        assertThat(counter.getCount()).isEqualTo(246L);
+    }
+
+    @Test
+    void testSuccessfulFileCountedWhenAnotherUploadFails() throws Exception {
+        Path file = Files.write(temporaryFolder.resolve("success.sst"), new byte[79]);
+        Counter counter = new ThreadSafeSimpleCounter();
+        KvSnapshotDataUploader uploader = new KvSnapshotDataUploader(uploaderThreadPool);
+        try (CloseableRegistry registry = new CloseableRegistry();
+                CloseableRegistry resources = new CloseableRegistry()) {
+            assertThatThrownBy(
+                            () ->
+                                    uploader.uploadFilesToSnapshotLocation(
+                                            Arrays.asList(
+                                                    file, temporaryFolder.resolve("missing.sst")),
+                                            createSnapshotLocation(),
+                                            SnapshotFileScope.SHARED,
+                                            registry,
+                                            resources,
+                                            counter))
+                    .isInstanceOf(IOException.class);
+            // The aggregate future fails fast; the other upload can still be running.
+            retry(Duration.ofMinutes(1), () -> assertThat(counter.getCount()).isEqualTo(79L));
+        }
+        assertThat(counter.getCount()).isEqualTo(79L);
+    }
+
+    @ParameterizedTest
+    @EnumSource(UploadFailure.class)
+    void testFailedUploadsNotCounted(UploadFailure failure) throws Exception {
+        Path file = Files.write(temporaryFolder.resolve("failed.sst"), new byte[123]);
+        Counter counter = new ThreadSafeSimpleCounter();
+        SnapshotLocation location = mock(SnapshotLocation.class);
+        SnapshotLocation.FsSnapshotOutputStream stream =
+                mock(SnapshotLocation.FsSnapshotOutputStream.class);
+        when(location.createSnapshotOutputStream(SnapshotFileScope.SHARED)).thenReturn(stream);
+        if (failure == UploadFailure.WRITE) {
+            doThrow(new IOException("write failed"))
+                    .when(stream)
+                    .write(any(byte[].class), anyInt(), anyInt());
+        } else {
+            when(stream.closeAndGetHandle()).thenThrow(new IOException("close failed"));
+        }
+        KvSnapshotDataUploader uploader = new KvSnapshotDataUploader(uploaderThreadPool);
+        try (CloseableRegistry registry = new CloseableRegistry();
+                CloseableRegistry resources = new CloseableRegistry()) {
+            assertThatThrownBy(
+                            () ->
+                                    uploader.uploadFilesToSnapshotLocation(
+                                            Collections.singletonList(file),
+                                            location,
+                                            SnapshotFileScope.SHARED,
+                                            registry,
+                                            resources,
+                                            counter))
+                    .isInstanceOf(IOException.class);
+            assertThat(counter.getCount()).isZero();
         }
     }
 
+    @Test
+    void testSuccessfulUploadCountedWhenCleanupRegistrationFails() throws Exception {
+        Path file = Files.write(temporaryFolder.resolve("success.sst"), new byte[91]);
+        Counter counter = new ThreadSafeSimpleCounter();
+        KvSnapshotDataUploader uploader = new KvSnapshotDataUploader(uploaderThreadPool);
+        try (CloseableRegistry registry = new CloseableRegistry();
+                CloseableRegistry resources = new CloseableRegistry()) {
+            resources.close();
+            assertThatThrownBy(
+                            () ->
+                                    uploader.uploadFilesToSnapshotLocation(
+                                            Collections.singletonList(file),
+                                            createSnapshotLocation(),
+                                            SnapshotFileScope.SHARED,
+                                            registry,
+                                            resources,
+                                            counter))
+                    .isInstanceOf(IOException.class);
+            assertThat(counter.getCount()).isEqualTo(91L);
+        }
+    }
+
+    @ParameterizedTest
+    @ValueSource(longs = {-1L, 9999L})
+    void testCountsCopiedBytesInsteadOfReportedHandleSize(long reportedSize) throws Exception {
+        Path file = Files.write(temporaryFolder.resolve("snapshot.sst"), new byte[123]);
+        Counter counter = new ThreadSafeSimpleCounter();
+        SnapshotLocation location = mock(SnapshotLocation.class);
+        SnapshotLocation.FsSnapshotOutputStream stream =
+                mock(SnapshotLocation.FsSnapshotOutputStream.class);
+        when(location.createSnapshotOutputStream(SnapshotFileScope.SHARED)).thenReturn(stream);
+        when(stream.closeAndGetHandle())
+                .thenReturn(
+                        new KvFileHandle(
+                                FsPath.fromLocalFile(
+                                                temporaryFolder.resolve("remote-file").toFile())
+                                        .toString(),
+                                reportedSize));
+        KvSnapshotDataUploader uploader = new KvSnapshotDataUploader(uploaderThreadPool);
+        try (CloseableRegistry registry = new CloseableRegistry();
+                CloseableRegistry resources = new CloseableRegistry()) {
+            uploader.uploadFilesToSnapshotLocation(
+                    Collections.singletonList(file),
+                    location,
+                    SnapshotFileScope.SHARED,
+                    registry,
+                    resources,
+                    counter);
+            assertThat(counter.getCount()).isEqualTo(123L);
+        }
+    }
+
+    private SnapshotLocation createSnapshotLocation() {
+        FsPath remote = FsPath.fromLocalFile(temporaryFolder.resolve("remote").toFile());
+        return new SnapshotLocation(LocalFileSystem.getSharedInstance(), remote, remote, 1024);
+    }
+
+    private enum UploadFailure {
+        WRITE,
+        CLOSE
+    }
+
     private void assertContentEqual(Path stateFilePath, FSDataInputStream inputStream)
             throws IOException {
         byte[] expected = Files.readAllBytes(stateFilePath);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvTabletSnapshotTargetTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvTabletSnapshotTargetTest.java
index 29773970e..00ab5dfd8 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvTabletSnapshotTargetTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvTabletSnapshotTargetTest.java
@@ -23,6 +23,8 @@ import org.apache.fluss.exception.FlussException;
 import org.apache.fluss.exception.FlussRuntimeException;
 import org.apache.fluss.fs.FsPath;
 import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metrics.Counter;
+import org.apache.fluss.metrics.ThreadSafeSimpleCounter;
 import org.apache.fluss.server.SequenceIDCounter;
 import org.apache.fluss.server.kv.rocksdb.RocksDBExtension;
 import org.apache.fluss.server.kv.rocksdb.RocksDBKv;
@@ -88,6 +90,7 @@ class KvTabletSnapshotTargetTest {
     private ManuallyTriggeredScheduledExecutorService scheduledExecutorService;
     private PeriodicSnapshotManager periodicSnapshotManager;
     private final CloseableRegistry closeableRegistry = new CloseableRegistry();
+    private final Counter remoteKvCopyBytes = new ThreadSafeSimpleCounter();
 
     private AtomicLong snapshotIdGenerator;
     private AtomicLong logOffsetGenerator;
@@ -245,6 +248,7 @@ class KvTabletSnapshotTargetTest {
     @Test
     void testAddToSnapshotToStoreFail(@TempDir Path kvTabletDir) throws Exception {
         final String errMsg = "Add to snapshot handle failed.";
+        AtomicLong uploadedBytes = new AtomicLong();
         AtomicBoolean shouldFail = new AtomicBoolean(true);
         // we use a store will fail when the variable shouldFail is true
         // add the snapshot to store will fail
@@ -252,6 +256,10 @@ class KvTabletSnapshotTargetTest {
                 TestCompletedSnapshotHandleStore.newBuilder()
                         .setAddFunction(
                                 (snapshot) -> {
+                                    uploadedBytes.addAndGet(
+                                            snapshot.retrieveCompleteSnapshot()
+                                                    .getKvSnapshotHandle()
+                                                    .getIncrementalSize());
                                     if (shouldFail.get()) {
                                         throw new FlussException(errMsg);
                                     }
@@ -286,6 +294,10 @@ class KvTabletSnapshotTargetTest {
         // minRetainOffset shouldn't be updated when the snapshot failed
         assertThat(updateMinRetainOffsetConsumer.get()).isEqualTo(Long.MAX_VALUE);
 
+        assertThat(uploadedBytes.get()).isPositive();
+        assertThat(remoteKvCopyBytes.getCount()).isEqualTo(uploadedBytes.get());
+        long bytesAfterFailedCommit = remoteKvCopyBytes.getCount();
+
         // set it to false
         shouldFail.set(false);
         long snapshotId2 = 2;
@@ -302,6 +314,9 @@ class KvTabletSnapshotTargetTest {
         assertThat(snapshotPath2.getFileSystem().exists(snapshotPath2)).isTrue();
         // minRetainOffset should be updated because snapshot success
         assertThat(updateMinRetainOffsetConsumer.get()).isEqualTo(1L);
+        assertThat(remoteKvCopyBytes.getCount())
+                .isEqualTo(uploadedBytes.get())
+                .isGreaterThan(bytesAfterFailedCommit);
     }
 
     @Test
@@ -571,7 +586,8 @@ class KvTabletSnapshotTargetTest {
                 snapshotDataUploader,
                 rocksDBExtension.getRockDbDir(),
                 lastCompletedSnapshotId,
-                snapshotFailType);
+                snapshotFailType,
+                remoteKvCopyBytes);
     }
 
     private ZooKeeperClient createFailingZooKeeperClient() {
@@ -620,14 +636,16 @@ class KvTabletSnapshotTargetTest {
                 KvSnapshotDataUploader kvSnapshotDataUploader,
                 @Nonnull File instanceBasePath,
                 long lastCompletedSnapshotId,
-                SnapshotFailType snapshotFailType) {
+                SnapshotFailType snapshotFailType,
+                Counter remoteKvCopyBytes) {
             super(
                     uploadedSstFiles,
                     db,
                     rocksDBResourceGuard,
                     kvSnapshotDataUploader,
                     instanceBasePath,
-                    lastCompletedSnapshotId);
+                    lastCompletedSnapshotId,
+                    remoteKvCopyBytes);
             this.snapshotFailType = snapshotFailType;
         }
 
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/RocksIncrementalSnapshotTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/RocksIncrementalSnapshotTest.java
index 64398516a..7446d9509 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/RocksIncrementalSnapshotTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/RocksIncrementalSnapshotTest.java
@@ -19,6 +19,8 @@ package org.apache.fluss.server.kv.snapshot;
 
 import org.apache.fluss.fs.FsPath;
 import org.apache.fluss.fs.local.LocalFileSystem;
+import org.apache.fluss.metrics.Counter;
+import org.apache.fluss.metrics.ThreadSafeSimpleCounter;
 import org.apache.fluss.server.kv.rocksdb.RocksDBExtension;
 import org.apache.fluss.server.kv.rocksdb.RocksDBKv;
 import org.apache.fluss.server.testutils.KvTestUtils;
@@ -40,6 +42,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.stream.Stream;
 
 import static org.apache.fluss.server.testutils.KvTestUtils.checkSnapshotIncrementWithNewlyFiles;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -51,6 +54,8 @@ class RocksIncrementalSnapshotTest {
 
     private static ExecutorService dataTransferThreadPool;
 
+    private final Counter remoteKvCopyBytes = new ThreadSafeSimpleCounter();
+
     @BeforeAll
     static void beforeAll() {
         dataTransferThreadPool = Executors.newSingleThreadExecutor();
@@ -102,7 +107,9 @@ class RocksIncrementalSnapshotTest {
             rocksDB.put("key2".getBytes(), "val2".getBytes());
             snapshot(3L, incrementalSnapshot, snapshotLocation, closeableRegistry);
             // assume it's fail
+            long bytesBeforeAbort = remoteKvCopyBytes.getCount();
             incrementalSnapshot.notifySnapshotAbort(3L);
+            assertThat(remoteKvCopyBytes.getCount()).isEqualTo(bytesBeforeAbort);
 
             // write some data again
             rocksDB.put("key3".getBytes(), "val3".getBytes());
@@ -177,9 +184,11 @@ class RocksIncrementalSnapshotTest {
                 rocksDBResourceGuard,
                 snapshotDataUploader,
                 rocksDBExtension.getRockDbDir(),
-                lastCompletedSnapshotId);
+                lastCompletedSnapshotId,
+                remoteKvCopyBytes);
     }
 
+    /** Takes a snapshot and verifies that only newly uploaded files increase the counter. */
     public KvSnapshotHandle snapshot(
             long snapshotId,
             RocksIncrementalSnapshot incrementalSnapshot,
@@ -189,13 +198,27 @@ class RocksIncrementalSnapshotTest {
         RocksIncrementalSnapshot.NativeRocksDBSnapshotResources nativeRocksDBSnapshotResources =
                 incrementalSnapshot.syncPrepareResources(snapshotId);
 
-        return incrementalSnapshot
-                .asyncSnapshot(
-                        nativeRocksDBSnapshotResources,
-                        snapshotId,
-                        new TabletState(0L, null, null),
-                        snapshotLocation)
-                .get(closeableRegistry)
-                .getKvSnapshotHandle();
+        long bytesBefore = remoteKvCopyBytes.getCount();
+        KvSnapshotHandle handle =
+                incrementalSnapshot
+                        .asyncSnapshot(
+                                nativeRocksDBSnapshotResources,
+                                snapshotId,
+                                new TabletState(0L, null, null),
+                                snapshotLocation)
+                        .get(closeableRegistry)
+                        .getKvSnapshotHandle();
+        long uploadedBytes =
+                Stream.concat(
+                                handle.getSharedKvFileHandles().stream(),
+                                handle.getPrivateFileHandles().stream())
+                        .filter(
+                                file ->
+                                        !(file.getKvFileHandle()
+                                                instanceof PlaceholderKvFileHandler))
+                        .mapToLong(file -> file.getKvFileHandle().getSize())
+                        .sum();
+        assertThat(remoteKvCopyBytes.getCount()).isEqualTo(bytesBefore + uploadedBytes);
+        return handle;
     }
 }
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/metrics/group/TableMetricGroupTest.java b/fluss-server/src/test/java/org/apache/fluss/server/metrics/group/TableMetricGroupTest.java
index 1ddf5c9a9..ad13f63a0 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/metrics/group/TableMetricGroupTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/metrics/group/TableMetricGroupTest.java
@@ -18,16 +18,91 @@
 package org.apache.fluss.server.metrics.group;
 
 import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.metrics.Counter;
+import org.apache.fluss.metrics.MeterView;
 import org.apache.fluss.metrics.MetricNames;
+import org.apache.fluss.metrics.ThreadSafeSimpleCounter;
+import org.apache.fluss.metrics.groups.AbstractMetricGroup;
+import org.apache.fluss.metrics.registry.MetricRegistry;
 import org.apache.fluss.metrics.registry.NOPMetricRegistry;
 
 import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
 
 /** Test for {@link TableMetricGroup}. */
 class TableMetricGroupTest {
 
+    @Test
+    void testRemoteKvCopyRateMetric() {
+        MetricRegistry registry = mock(MetricRegistry.class);
+        TableMetricGroup group =
+                new TableMetricGroup(
+                        registry,
+                        TablePath.of("db", "pk_table"),
+                        true,
+                        TestingMetricGroups.TABLET_SERVER_METRICS);
+        try {
+            Counter counter = group.remoteKvCopyBytes();
+            assertThat(counter).isInstanceOf(ThreadSafeSimpleCounter.class);
+            ArgumentCaptor<AbstractMetricGroup> metricGroup =
+                    ArgumentCaptor.forClass(AbstractMetricGroup.class);
+            ArgumentCaptor<MeterView> meter = ArgumentCaptor.forClass(MeterView.class);
+            verify(registry)
+                    .register(
+                            meter.capture(),
+                            eq(MetricNames.REMOTE_KV_COPY_BYTES_RATE),
+                            metricGroup.capture());
+            verify(registry, never()).register(eq(counter), any(), any());
+            assertThat(metricGroup.getValue().getMetrics())
+                    .containsEntry(MetricNames.REMOTE_KV_COPY_BYTES_RATE, meter.getValue())
+                    .doesNotContainKey("remoteKvCopyBytes");
+            assertThat(metricGroup.getValue().getAllVariables())
+                    .containsEntry("database", "db")
+                    .containsEntry("table", "pk_table");
+            assertThat(metricGroup.getValue().getScopeComponents()).endsWith("kv");
+            counter.inc(120L);
+            assertThat(meter.getValue().getCount()).isEqualTo(120L);
+            meter.getValue().update();
+            assertThat(meter.getValue().getRate()).isEqualTo(2.0);
+            group.close();
+            verify(registry, never()).unregister(eq(counter), any(), any());
+            verify(registry)
+                    .unregister(
+                            meter.getValue(),
+                            MetricNames.REMOTE_KV_COPY_BYTES_RATE,
+                            metricGroup.getValue());
+        } finally {
+            group.close();
+        }
+    }
+
+    @Test
+    void testLogTableDoesNotExposeRemoteKvCopyRateMetric() {
+        MetricRegistry registry = mock(MetricRegistry.class);
+        TableMetricGroup group =
+                new TableMetricGroup(
+                        registry,
+                        TablePath.of("db", "log_table"),
+                        false,
+                        TestingMetricGroups.TABLET_SERVER_METRICS);
+        try {
+            group.remoteKvCopyBytes().inc(120L);
+            assertThat(group.remoteKvCopyBytes().getCount()).isZero();
+            verify(registry, never()).register(any(), eq("remoteKvCopyBytes"), any());
+            verify(registry, never())
+                    .register(any(), eq(MetricNames.REMOTE_KV_COPY_BYTES_RATE), any());
+        } finally {
+            group.close();
+        }
+    }
+
     @Test
     void testKvBackpressureMetricsExposeOnlyCoreSignals() {
         TableMetricGroup metricGroup =
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java
index b9e8cb7ce..50a9dc79e 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java
@@ -612,6 +612,7 @@ final class ReplicaTest extends ReplicaTestBase {
                         Tuple2.of("k2", new Object[] {3, "b1"}));
         putRecordsToLeader(kvReplica, kvRecords);
 
+        long uploadedBytesBefore = kvReplica.tableMetrics().remoteKvCopyBytes().getCount();
         // trigger one snapshot (task has been scheduled after becoming leader)
         scheduledExecutorService.triggerAllNonPeriodicTasks();
 
@@ -627,6 +628,10 @@ final class ReplicaTest extends ReplicaTestBase {
                                 Tuple2.of("k1", new Object[] {2, "b"}),
                                 Tuple2.of("k2", new Object[] {3, "b1"})));
         KvTestUtils.checkSnapshot(completedSnapshot0, expectedKeyValues, expectedLogOffset);
+        assertThat(kvReplica.tableMetrics().remoteKvCopyBytes().getCount())
+                .isEqualTo(
+                        uploadedBytesBefore
+                                + completedSnapshot0.getKvSnapshotHandle().getIncrementalSize());
 
         // put some data again
         kvRecords =
@@ -644,6 +649,11 @@ final class ReplicaTest extends ReplicaTestBase {
         expectedKeyValues =
                 getKeyValuePairs(genKvRecords(Tuple2.of("k2", new Object[] {4, "bk2"})));
         KvTestUtils.checkSnapshot(completedSnapshot1, expectedKeyValues, expectedLogOffset);
+        assertThat(kvReplica.tableMetrics().remoteKvCopyBytes().getCount())
+                .isEqualTo(
+                        uploadedBytesBefore
+                                + completedSnapshot0.getKvSnapshotHandle().getIncrementalSize()
+                                + completedSnapshot1.getKvSnapshotHandle().getIncrementalSize());
 
         // check the snapshot should be incremental, with only one newly file
         KvTestUtils.checkSnapshotIncrementWithNewlyFiles(
diff --git a/website/docs/maintenance/observability/monitor-metrics.md b/website/docs/maintenance/observability/monitor-metrics.md
index 3cbc4e293..ee57382f5 100644
--- a/website/docs/maintenance/observability/monitor-metrics.md
+++ b/website/docs/maintenance/observability/monitor-metrics.md
@@ -806,8 +806,8 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM
   </thead>
   <tbody>
     <tr>
-      <th rowspan="39"><strong>tabletserver</strong></th>
-      <td rowspan="20">table</td>
+      <th rowspan="41"><strong>tabletserver</strong></th>
+      <td rowspan="22">table</td>
       <td>messagesInPerSecond</td>
       <td>The number of messages written per second to this table.</td>
       <td>Meter</td>
@@ -907,6 +907,11 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM
       <td>The number of failed delete remote log requests to delete remote log after log ttl per second.</td>
       <td>Meter</td>
     </tr>
+    <tr>
+      <td>remoteKvCopyBytesPerSecond</td>
+      <td>The bytes of kv snapshot data uploaded to remote per second for this table. Only available for primary key tables.</td>
+      <td>Meter</td>
+    </tr>
     <tr>
       <td rowspan="6">table_historical</td>
       <td>totalPutKvRequestsPerSecond</td>