You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by ya...@apache.org on 2026/09/17 05:53:24 UTC
(fluss) branch main updated: [metrics] Add KV snapshot upload throughput metrics
This is an automated email from the ASF dual-hosted git repository.
platinumhamburg pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
The following commit(s) were added to refs/heads/main by this push:
new aade48b0d [metrics] Add KV snapshot upload throughput metrics
aade48b0d is described below
commit aade48b0dd6f968015d2c3e183242d513c1127be
Author: chaohaochen <57...@users.noreply.github.com>
AuthorDate: Thu Sep 17 13:53:20 2026 +0800
[metrics] Add KV snapshot upload throughput metrics
Co-authored-by: 韵成 <ch...@alibaba-inc.com>
---
.../java/org/apache/fluss/metrics/MetricNames.java | 2 +
.../java/org/apache/fluss/server/kv/KvTablet.java | 8 +-
.../server/kv/snapshot/KvSnapshotDataUploader.java | 23 ++-
.../kv/snapshot/RocksIncrementalSnapshot.java | 14 +-
.../server/metrics/group/TableMetricGroup.java | 15 ++
.../org/apache/fluss/server/replica/Replica.java | 3 +-
.../kv/snapshot/KvSnapshotDataUploaderTest.java | 199 ++++++++++++++++++++-
.../kv/snapshot/KvTabletSnapshotTargetTest.java | 24 ++-
.../kv/snapshot/RocksIncrementalSnapshotTest.java | 41 ++++-
.../server/metrics/group/TableMetricGroupTest.java | 75 ++++++++
.../apache/fluss/server/replica/ReplicaTest.java | 10 ++
.../maintenance/observability/monitor-metrics.md | 9 +-
12 files changed, 389 insertions(+), 34 deletions(-)
diff --git a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java
index ac60ee5f1..0219f3fe7 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java
@@ -135,6 +135,8 @@ public class MetricNames {
public static final String FAILED_PRODUCE_FETCH_LOG_REQUESTS_RATE =
"failedProduceLogRequestsPerSecond";
+ public static final String REMOTE_KV_COPY_BYTES_RATE = "remoteKvCopyBytesPerSecond";
+
public static final String REMOTE_LOG_COPY_BYTES_RATE = "remoteLogCopyBytesPerSecond";
public static final String REMOTE_LOG_COPY_REQUESTS_RATE = "remoteLogCopyRequestsPerSecond";
public static final String REMOTE_LOG_COPY_ERROR_RATE = "remoteLogCopyErrorPerSecond";
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
index 2312b6fd4..61c5a2d5d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
@@ -32,6 +32,7 @@ import org.apache.fluss.metadata.PhysicalTablePath;
import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.metrics.Counter;
import org.apache.fluss.record.ChangeType;
import org.apache.fluss.record.KvRecordBatch;
import org.apache.fluss.row.arrow.ArrowWriterPool;
@@ -1369,17 +1370,20 @@ public final class KvTablet {
});
}
+ /** Creates an incremental snapshot with the table's thread-safe upload byte counter. */
public RocksIncrementalSnapshot createIncrementalSnapshot(
Map<Long, Collection<KvFileHandleAndLocalPath>> uploadedSstFiles,
KvSnapshotDataUploader kvSnapshotDataUploader,
- long lastCompletedSnapshotId) {
+ long lastCompletedSnapshotId,
+ Counter remoteKvCopyBytes) {
return new RocksIncrementalSnapshot(
uploadedSstFiles,
rocksDBKv.getDb(),
rocksDBKv.getResourceGuard(),
kvSnapshotDataUploader,
kvTabletDir,
- lastCompletedSnapshotId);
+ lastCompletedSnapshotId,
+ remoteKvCopyBytes);
}
// only for testing.
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/KvSnapshotDataUploader.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/KvSnapshotDataUploader.java
index ef4d3b6ee..e0f958ec6 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/KvSnapshotDataUploader.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/KvSnapshotDataUploader.java
@@ -18,6 +18,7 @@
package org.apache.fluss.server.kv.snapshot;
import org.apache.fluss.exception.FlussRuntimeException;
+import org.apache.fluss.metrics.Counter;
import org.apache.fluss.server.utils.SnapshotUtil;
import org.apache.fluss.utils.CloseableRegistry;
import org.apache.fluss.utils.ExceptionUtils;
@@ -51,6 +52,7 @@ public class KvSnapshotDataUploader extends KvSnapshotDataTransfer {
* Upload all the files to the target snapshot location using specified number of threads.
*
* @param files The files will be uploaded to the snapshot location.
+ * @param remoteKvCopyBytes Thread-safe counter for successfully uploaded bytes of this table.
* @throws Exception Thrown if can not upload all the files.
*/
public List<KvFileHandleAndLocalPath> uploadFilesToSnapshotLocation(
@@ -58,7 +60,8 @@ public class KvSnapshotDataUploader extends KvSnapshotDataTransfer {
SnapshotLocation snapshotLocation,
SnapshotFileScope snapshotFileScope,
CloseableRegistry closeableRegistry,
- CloseableRegistry tmpResourcesRegistry)
+ CloseableRegistry tmpResourcesRegistry,
+ Counter remoteKvCopyBytes)
throws Exception {
List<CompletableFuture<KvFileHandleAndLocalPath>> futures =
@@ -67,7 +70,8 @@ public class KvSnapshotDataUploader extends KvSnapshotDataTransfer {
snapshotLocation,
snapshotFileScope,
closeableRegistry,
- tmpResourcesRegistry);
+ tmpResourcesRegistry,
+ remoteKvCopyBytes);
List<KvFileHandleAndLocalPath> handles = new ArrayList<>(files.size());
@@ -95,7 +99,8 @@ public class KvSnapshotDataUploader extends KvSnapshotDataTransfer {
SnapshotLocation snapshotLocation,
SnapshotFileScope snapshotFileScope,
CloseableRegistry closeableRegistry,
- CloseableRegistry tmpResourcesRegistry) {
+ CloseableRegistry tmpResourcesRegistry,
+ Counter remoteKvCopyBytes) {
return files.stream()
.map(
e ->
@@ -107,7 +112,8 @@ public class KvSnapshotDataUploader extends KvSnapshotDataTransfer {
snapshotLocation,
snapshotFileScope,
closeableRegistry,
- tmpResourcesRegistry)),
+ tmpResourcesRegistry,
+ remoteKvCopyBytes)),
dataTransferThreadPool))
.collect(Collectors.toList());
}
@@ -117,7 +123,8 @@ public class KvSnapshotDataUploader extends KvSnapshotDataTransfer {
SnapshotLocation snapshotLocation,
SnapshotFileScope snapshotFileScope,
CloseableRegistry closeableRegistry,
- CloseableRegistry tmpResourcesRegistry)
+ CloseableRegistry tmpResourcesRegistry,
+ Counter remoteKvCopyBytes)
throws IOException {
InputStream inputStream = null;
@@ -132,6 +139,7 @@ public class KvSnapshotDataUploader extends KvSnapshotDataTransfer {
outputStream = snapshotLocation.createSnapshotOutputStream(snapshotFileScope);
closeableRegistry.registerCloseable(outputStream);
+ long uploadedBytes = 0L;
while (true) {
int numBytes = inputStream.read(buffer);
@@ -139,11 +147,16 @@ public class KvSnapshotDataUploader extends KvSnapshotDataTransfer {
break;
}
outputStream.write(buffer, 0, numBytes);
+ uploadedBytes += numBytes;
}
final KvFileHandle result;
if (closeableRegistry.unregisterCloseable(outputStream)) {
result = outputStream.closeAndGetHandle();
+ if (result != null) {
+ // Count the uploaded bytes even if this snapshot later fails or aborts.
+ remoteKvCopyBytes.inc(uploadedBytes);
+ }
} else {
result = null;
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/RocksIncrementalSnapshot.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/RocksIncrementalSnapshot.java
index c50fe3ef0..9cde2d5d8 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/RocksIncrementalSnapshot.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/RocksIncrementalSnapshot.java
@@ -17,6 +17,7 @@
package org.apache.fluss.server.kv.snapshot;
+import org.apache.fluss.metrics.Counter;
import org.apache.fluss.server.utils.ResourceGuard;
import org.apache.fluss.utils.CloseableRegistry;
import org.apache.fluss.utils.ExceptionUtils;
@@ -80,13 +81,18 @@ public class RocksIncrementalSnapshot implements AutoCloseable {
/** The help class used to upload kv snapshot files. */
private final KvSnapshotDataUploader kvSnapshotDataUploader;
+ private final Counter remoteKvCopyBytes;
+
+ /** Creates an incremental snapshot using the table's thread-safe upload byte counter. */
public RocksIncrementalSnapshot(
Map<Long, Collection<KvFileHandleAndLocalPath>> uploadedSstFiles,
@Nonnull RocksDB db,
ResourceGuard rocksDBResourceGuard,
KvSnapshotDataUploader kvSnapshotDataUploader,
@Nonnull File instanceBasePath,
- long lastCompletedSnapshotId) {
+ long lastCompletedSnapshotId,
+ Counter remoteKvCopyBytes) {
+ this.remoteKvCopyBytes = remoteKvCopyBytes;
this.uploadedSstFiles = uploadedSstFiles;
this.db = db;
this.rocksDBResourceGuard = rocksDBResourceGuard;
@@ -253,7 +259,8 @@ public class RocksIncrementalSnapshot implements AutoCloseable {
snapshotLocation,
SnapshotFileScope.SHARED,
closeableRegistry,
- tmpResourcesRegistry);
+ tmpResourcesRegistry,
+ remoteKvCopyBytes);
size +=
sstFilesUploadResult.stream()
.mapToLong(e -> e.getKvFileHandle().getSize())
@@ -268,7 +275,8 @@ public class RocksIncrementalSnapshot implements AutoCloseable {
snapshotLocation,
SnapshotFileScope.EXCLUSIVE,
closeableRegistry,
- tmpResourcesRegistry);
+ tmpResourcesRegistry,
+ remoteKvCopyBytes);
size +=
miscFilesUploadResult.stream()
.mapToLong(e -> e.getKvFileHandle().getSize())
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java
index 02be54ed9..523045872 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java
@@ -169,6 +169,15 @@ public class TableMetricGroup extends AbstractMetricGroup {
return logMetrics.remoteLogDeleteErrors;
}
+ /** Returns the successful KV snapshot upload byte counter, or a no-op for log tables. */
+ public Counter remoteKvCopyBytes() {
+ if (kvMetrics == null) {
+ return NoOpCounter.INSTANCE;
+ } else {
+ return kvMetrics.remoteKvCopyBytes;
+ }
+ }
+
public void incKvMessageIn(long n) {
if (kvMetrics == null) {
NoOpCounter.INSTANCE.inc(n);
@@ -592,6 +601,8 @@ public class TableMetricGroup extends AbstractMetricGroup {
private static final String LOOKUP_FILE_DOWNLOADED = "lookup_file_downloaded";
+ private final Counter remoteKvCopyBytes;
+
private final Counter totalLookupRequests;
private final Counter failedLookupRequests;
private final Counter totalHistoricalLookupRequests;
@@ -610,6 +621,10 @@ public class TableMetricGroup extends AbstractMetricGroup {
public KvMetricGroup(TableMetricGroup tableMetricGroup) {
super(tableMetricGroup, TabletType.KV);
+ // for kv snapshot upload
+ remoteKvCopyBytes = new ThreadSafeSimpleCounter();
+ meter(MetricNames.REMOTE_KV_COPY_BYTES_RATE, new MeterView(remoteKvCopyBytes));
+
// for lookup request
totalLookupRequests = new ThreadSafeSimpleCounter();
meter(MetricNames.TOTAL_LOOKUP_REQUESTS_RATE, new MeterView(totalLookupRequests));
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
index a74056ebc..1c33025fc 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
@@ -1131,7 +1131,8 @@ public final class Replica {
kvTablet.createIncrementalSnapshot(
uploadedSstFiles,
snapshotContext.getSnapshotDataUploader(),
- lastCompletedSnapshotId);
+ lastCompletedSnapshotId,
+ tableMetrics().remoteKvCopyBytes());
// create snapshot ID counter
SequenceIDCounter snapshotIDCounter =
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvSnapshotDataUploaderTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvSnapshotDataUploaderTest.java
index ef5048593..9432b9abc 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvSnapshotDataUploaderTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvSnapshotDataUploaderTest.java
@@ -20,6 +20,8 @@ package org.apache.fluss.server.kv.snapshot;
import org.apache.fluss.fs.FSDataInputStream;
import org.apache.fluss.fs.FsPath;
import org.apache.fluss.fs.local.LocalFileSystem;
+import org.apache.fluss.metrics.Counter;
+import org.apache.fluss.metrics.ThreadSafeSimpleCounter;
import org.apache.fluss.utils.CloseableRegistry;
import org.apache.fluss.utils.IOUtils;
@@ -27,39 +29,52 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.ValueSource;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.time.Duration;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadLocalRandom;
+import static org.apache.fluss.testutils.common.CommonTestUtils.retry;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
/** Test class for {@link org.apache.fluss.server.kv.snapshot.KvSnapshotDataUploader}. */
class KvSnapshotDataUploaderTest {
@TempDir private Path temporaryFolder;
- private ExecutorService downLoaderThreadPool;
+ private ExecutorService uploaderThreadPool;
@BeforeEach
void beforeEach() {
- downLoaderThreadPool = Executors.newSingleThreadExecutor();
+ uploaderThreadPool = Executors.newFixedThreadPool(4);
}
@AfterEach
void afterEach() {
- if (downLoaderThreadPool != null) {
- downLoaderThreadPool.shutdownNow();
+ if (uploaderThreadPool != null) {
+ uploaderThreadPool.shutdownNow();
}
}
- /** Test that the exception arose in the thread pool will rethrow to the main thread. */
+ /** Verifies file contents and byte accounting across concurrent uploads. */
@Test
void testMultiThreadUploadCorrectly() throws Exception {
File snapshotSharedFolder = new File(temporaryFolder.toFile(), "shared");
@@ -80,16 +95,20 @@ class KvSnapshotDataUploaderTest {
List<Path> sstFilePaths =
generateRandomSstFiles(localFolder, sstFileCount, fileSizeThreshold);
- KvSnapshotDataUploader snapshotUploader = new KvSnapshotDataUploader(downLoaderThreadPool);
+ KvSnapshotDataUploader snapshotUploader = new KvSnapshotDataUploader(uploaderThreadPool);
+ Counter counter = new ThreadSafeSimpleCounter();
List<KvFileHandleAndLocalPath> sstFiles =
snapshotUploader.uploadFilesToSnapshotLocation(
sstFilePaths,
snapshotLocation,
SnapshotFileScope.SHARED,
new CloseableRegistry(),
- new CloseableRegistry());
+ new CloseableRegistry(),
+ counter);
+ long expectedBytes = 0L;
for (Path path : sstFilePaths) {
+ expectedBytes += Files.size(path);
KvFileHandle kvFileHandle =
sstFiles.stream()
.filter(e -> e.getLocalPath().equals(path.getFileName().toString()))
@@ -99,11 +118,173 @@ class KvSnapshotDataUploaderTest {
assertThat(kvFileHandle.getFilePath())
.startsWith(LocalFileSystem.getLocalFsURI().getScheme());
FsPath fsPath = new FsPath(kvFileHandle.getFilePath());
- FSDataInputStream inputStream = fsPath.getFileSystem().open(fsPath);
- assertContentEqual(path, inputStream);
+ try (FSDataInputStream inputStream = fsPath.getFileSystem().open(fsPath)) {
+ assertContentEqual(path, inputStream);
+ }
+ }
+ assertThat(counter.getCount()).isEqualTo(expectedBytes);
+ }
+
+ @ParameterizedTest
+ @EnumSource(SnapshotFileScope.class)
+ void testSuccessfulUploadsAndReuploads(SnapshotFileScope scope) throws Exception {
+ Path file = Files.write(temporaryFolder.resolve("snapshot-file"), new byte[123]);
+ Counter counter = new ThreadSafeSimpleCounter();
+ Counter otherTableCounter = new ThreadSafeSimpleCounter();
+ KvSnapshotDataUploader uploader = new KvSnapshotDataUploader(uploaderThreadPool);
+ try (CloseableRegistry registry = new CloseableRegistry();
+ CloseableRegistry resources = new CloseableRegistry()) {
+ for (int i = 1; i <= 2; i++) {
+ uploader.uploadFilesToSnapshotLocation(
+ Collections.singletonList(file),
+ createSnapshotLocation(),
+ scope,
+ registry,
+ resources,
+ counter);
+ assertThat(counter.getCount()).isEqualTo(123L * i);
+ }
+ uploader.uploadFilesToSnapshotLocation(
+ Collections.singletonList(file),
+ createSnapshotLocation(),
+ scope,
+ registry,
+ resources,
+ otherTableCounter);
+ assertThat(otherTableCounter.getCount()).isEqualTo(123L);
+ assertThat(counter.getCount()).isEqualTo(246L);
+ uploader.uploadFilesToSnapshotLocation(
+ Collections.emptyList(),
+ createSnapshotLocation(),
+ scope,
+ registry,
+ resources,
+ counter);
+ assertThat(counter.getCount()).isEqualTo(246L);
+ }
+ // Cleanup of an aborted snapshot must not roll back successful upload traffic.
+ assertThat(counter.getCount()).isEqualTo(246L);
+ }
+
+ @Test
+ void testSuccessfulFileCountedWhenAnotherUploadFails() throws Exception {
+ Path file = Files.write(temporaryFolder.resolve("success.sst"), new byte[79]);
+ Counter counter = new ThreadSafeSimpleCounter();
+ KvSnapshotDataUploader uploader = new KvSnapshotDataUploader(uploaderThreadPool);
+ try (CloseableRegistry registry = new CloseableRegistry();
+ CloseableRegistry resources = new CloseableRegistry()) {
+ assertThatThrownBy(
+ () ->
+ uploader.uploadFilesToSnapshotLocation(
+ Arrays.asList(
+ file, temporaryFolder.resolve("missing.sst")),
+ createSnapshotLocation(),
+ SnapshotFileScope.SHARED,
+ registry,
+ resources,
+ counter))
+ .isInstanceOf(IOException.class);
+ // The aggregate future fails fast; the other upload can still be running.
+ retry(Duration.ofMinutes(1), () -> assertThat(counter.getCount()).isEqualTo(79L));
+ }
+ assertThat(counter.getCount()).isEqualTo(79L);
+ }
+
+ @ParameterizedTest
+ @EnumSource(UploadFailure.class)
+ void testFailedUploadsNotCounted(UploadFailure failure) throws Exception {
+ Path file = Files.write(temporaryFolder.resolve("failed.sst"), new byte[123]);
+ Counter counter = new ThreadSafeSimpleCounter();
+ SnapshotLocation location = mock(SnapshotLocation.class);
+ SnapshotLocation.FsSnapshotOutputStream stream =
+ mock(SnapshotLocation.FsSnapshotOutputStream.class);
+ when(location.createSnapshotOutputStream(SnapshotFileScope.SHARED)).thenReturn(stream);
+ if (failure == UploadFailure.WRITE) {
+ doThrow(new IOException("write failed"))
+ .when(stream)
+ .write(any(byte[].class), anyInt(), anyInt());
+ } else {
+ when(stream.closeAndGetHandle()).thenThrow(new IOException("close failed"));
+ }
+ KvSnapshotDataUploader uploader = new KvSnapshotDataUploader(uploaderThreadPool);
+ try (CloseableRegistry registry = new CloseableRegistry();
+ CloseableRegistry resources = new CloseableRegistry()) {
+ assertThatThrownBy(
+ () ->
+ uploader.uploadFilesToSnapshotLocation(
+ Collections.singletonList(file),
+ location,
+ SnapshotFileScope.SHARED,
+ registry,
+ resources,
+ counter))
+ .isInstanceOf(IOException.class);
+ assertThat(counter.getCount()).isZero();
}
}
+ @Test
+ void testSuccessfulUploadCountedWhenCleanupRegistrationFails() throws Exception {
+ Path file = Files.write(temporaryFolder.resolve("success.sst"), new byte[91]);
+ Counter counter = new ThreadSafeSimpleCounter();
+ KvSnapshotDataUploader uploader = new KvSnapshotDataUploader(uploaderThreadPool);
+ try (CloseableRegistry registry = new CloseableRegistry();
+ CloseableRegistry resources = new CloseableRegistry()) {
+ resources.close();
+ assertThatThrownBy(
+ () ->
+ uploader.uploadFilesToSnapshotLocation(
+ Collections.singletonList(file),
+ createSnapshotLocation(),
+ SnapshotFileScope.SHARED,
+ registry,
+ resources,
+ counter))
+ .isInstanceOf(IOException.class);
+ assertThat(counter.getCount()).isEqualTo(91L);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(longs = {-1L, 9999L})
+ void testCountsCopiedBytesInsteadOfReportedHandleSize(long reportedSize) throws Exception {
+ Path file = Files.write(temporaryFolder.resolve("snapshot.sst"), new byte[123]);
+ Counter counter = new ThreadSafeSimpleCounter();
+ SnapshotLocation location = mock(SnapshotLocation.class);
+ SnapshotLocation.FsSnapshotOutputStream stream =
+ mock(SnapshotLocation.FsSnapshotOutputStream.class);
+ when(location.createSnapshotOutputStream(SnapshotFileScope.SHARED)).thenReturn(stream);
+ when(stream.closeAndGetHandle())
+ .thenReturn(
+ new KvFileHandle(
+ FsPath.fromLocalFile(
+ temporaryFolder.resolve("remote-file").toFile())
+ .toString(),
+ reportedSize));
+ KvSnapshotDataUploader uploader = new KvSnapshotDataUploader(uploaderThreadPool);
+ try (CloseableRegistry registry = new CloseableRegistry();
+ CloseableRegistry resources = new CloseableRegistry()) {
+ uploader.uploadFilesToSnapshotLocation(
+ Collections.singletonList(file),
+ location,
+ SnapshotFileScope.SHARED,
+ registry,
+ resources,
+ counter);
+ assertThat(counter.getCount()).isEqualTo(123L);
+ }
+ }
+
+ private SnapshotLocation createSnapshotLocation() {
+ FsPath remote = FsPath.fromLocalFile(temporaryFolder.resolve("remote").toFile());
+ return new SnapshotLocation(LocalFileSystem.getSharedInstance(), remote, remote, 1024);
+ }
+
+ private enum UploadFailure {
+ WRITE,
+ CLOSE
+ }
+
private void assertContentEqual(Path stateFilePath, FSDataInputStream inputStream)
throws IOException {
byte[] expected = Files.readAllBytes(stateFilePath);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvTabletSnapshotTargetTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvTabletSnapshotTargetTest.java
index 29773970e..00ab5dfd8 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvTabletSnapshotTargetTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvTabletSnapshotTargetTest.java
@@ -23,6 +23,8 @@ import org.apache.fluss.exception.FlussException;
import org.apache.fluss.exception.FlussRuntimeException;
import org.apache.fluss.fs.FsPath;
import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metrics.Counter;
+import org.apache.fluss.metrics.ThreadSafeSimpleCounter;
import org.apache.fluss.server.SequenceIDCounter;
import org.apache.fluss.server.kv.rocksdb.RocksDBExtension;
import org.apache.fluss.server.kv.rocksdb.RocksDBKv;
@@ -88,6 +90,7 @@ class KvTabletSnapshotTargetTest {
private ManuallyTriggeredScheduledExecutorService scheduledExecutorService;
private PeriodicSnapshotManager periodicSnapshotManager;
private final CloseableRegistry closeableRegistry = new CloseableRegistry();
+ private final Counter remoteKvCopyBytes = new ThreadSafeSimpleCounter();
private AtomicLong snapshotIdGenerator;
private AtomicLong logOffsetGenerator;
@@ -245,6 +248,7 @@ class KvTabletSnapshotTargetTest {
@Test
void testAddToSnapshotToStoreFail(@TempDir Path kvTabletDir) throws Exception {
final String errMsg = "Add to snapshot handle failed.";
+ AtomicLong uploadedBytes = new AtomicLong();
AtomicBoolean shouldFail = new AtomicBoolean(true);
// we use a store will fail when the variable shouldFail is true
// add the snapshot to store will fail
@@ -252,6 +256,10 @@ class KvTabletSnapshotTargetTest {
TestCompletedSnapshotHandleStore.newBuilder()
.setAddFunction(
(snapshot) -> {
+ uploadedBytes.addAndGet(
+ snapshot.retrieveCompleteSnapshot()
+ .getKvSnapshotHandle()
+ .getIncrementalSize());
if (shouldFail.get()) {
throw new FlussException(errMsg);
}
@@ -286,6 +294,10 @@ class KvTabletSnapshotTargetTest {
// minRetainOffset shouldn't be updated when the snapshot failed
assertThat(updateMinRetainOffsetConsumer.get()).isEqualTo(Long.MAX_VALUE);
+ assertThat(uploadedBytes.get()).isPositive();
+ assertThat(remoteKvCopyBytes.getCount()).isEqualTo(uploadedBytes.get());
+ long bytesAfterFailedCommit = remoteKvCopyBytes.getCount();
+
// set it to false
shouldFail.set(false);
long snapshotId2 = 2;
@@ -302,6 +314,9 @@ class KvTabletSnapshotTargetTest {
assertThat(snapshotPath2.getFileSystem().exists(snapshotPath2)).isTrue();
// minRetainOffset should be updated because snapshot success
assertThat(updateMinRetainOffsetConsumer.get()).isEqualTo(1L);
+ assertThat(remoteKvCopyBytes.getCount())
+ .isEqualTo(uploadedBytes.get())
+ .isGreaterThan(bytesAfterFailedCommit);
}
@Test
@@ -571,7 +586,8 @@ class KvTabletSnapshotTargetTest {
snapshotDataUploader,
rocksDBExtension.getRockDbDir(),
lastCompletedSnapshotId,
- snapshotFailType);
+ snapshotFailType,
+ remoteKvCopyBytes);
}
private ZooKeeperClient createFailingZooKeeperClient() {
@@ -620,14 +636,16 @@ class KvTabletSnapshotTargetTest {
KvSnapshotDataUploader kvSnapshotDataUploader,
@Nonnull File instanceBasePath,
long lastCompletedSnapshotId,
- SnapshotFailType snapshotFailType) {
+ SnapshotFailType snapshotFailType,
+ Counter remoteKvCopyBytes) {
super(
uploadedSstFiles,
db,
rocksDBResourceGuard,
kvSnapshotDataUploader,
instanceBasePath,
- lastCompletedSnapshotId);
+ lastCompletedSnapshotId,
+ remoteKvCopyBytes);
this.snapshotFailType = snapshotFailType;
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/RocksIncrementalSnapshotTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/RocksIncrementalSnapshotTest.java
index 64398516a..7446d9509 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/RocksIncrementalSnapshotTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/RocksIncrementalSnapshotTest.java
@@ -19,6 +19,8 @@ package org.apache.fluss.server.kv.snapshot;
import org.apache.fluss.fs.FsPath;
import org.apache.fluss.fs.local.LocalFileSystem;
+import org.apache.fluss.metrics.Counter;
+import org.apache.fluss.metrics.ThreadSafeSimpleCounter;
import org.apache.fluss.server.kv.rocksdb.RocksDBExtension;
import org.apache.fluss.server.kv.rocksdb.RocksDBKv;
import org.apache.fluss.server.testutils.KvTestUtils;
@@ -40,6 +42,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.stream.Stream;
import static org.apache.fluss.server.testutils.KvTestUtils.checkSnapshotIncrementWithNewlyFiles;
import static org.assertj.core.api.Assertions.assertThat;
@@ -51,6 +54,8 @@ class RocksIncrementalSnapshotTest {
private static ExecutorService dataTransferThreadPool;
+ private final Counter remoteKvCopyBytes = new ThreadSafeSimpleCounter();
+
@BeforeAll
static void beforeAll() {
dataTransferThreadPool = Executors.newSingleThreadExecutor();
@@ -102,7 +107,9 @@ class RocksIncrementalSnapshotTest {
rocksDB.put("key2".getBytes(), "val2".getBytes());
snapshot(3L, incrementalSnapshot, snapshotLocation, closeableRegistry);
// assume it's fail
+ long bytesBeforeAbort = remoteKvCopyBytes.getCount();
incrementalSnapshot.notifySnapshotAbort(3L);
+ assertThat(remoteKvCopyBytes.getCount()).isEqualTo(bytesBeforeAbort);
// write some data again
rocksDB.put("key3".getBytes(), "val3".getBytes());
@@ -177,9 +184,11 @@ class RocksIncrementalSnapshotTest {
rocksDBResourceGuard,
snapshotDataUploader,
rocksDBExtension.getRockDbDir(),
- lastCompletedSnapshotId);
+ lastCompletedSnapshotId,
+ remoteKvCopyBytes);
}
+ /** Takes a snapshot and verifies that only newly uploaded files increase the counter. */
public KvSnapshotHandle snapshot(
long snapshotId,
RocksIncrementalSnapshot incrementalSnapshot,
@@ -189,13 +198,27 @@ class RocksIncrementalSnapshotTest {
RocksIncrementalSnapshot.NativeRocksDBSnapshotResources nativeRocksDBSnapshotResources =
incrementalSnapshot.syncPrepareResources(snapshotId);
- return incrementalSnapshot
- .asyncSnapshot(
- nativeRocksDBSnapshotResources,
- snapshotId,
- new TabletState(0L, null, null),
- snapshotLocation)
- .get(closeableRegistry)
- .getKvSnapshotHandle();
+ long bytesBefore = remoteKvCopyBytes.getCount();
+ KvSnapshotHandle handle =
+ incrementalSnapshot
+ .asyncSnapshot(
+ nativeRocksDBSnapshotResources,
+ snapshotId,
+ new TabletState(0L, null, null),
+ snapshotLocation)
+ .get(closeableRegistry)
+ .getKvSnapshotHandle();
+ long uploadedBytes =
+ Stream.concat(
+ handle.getSharedKvFileHandles().stream(),
+ handle.getPrivateFileHandles().stream())
+ .filter(
+ file ->
+ !(file.getKvFileHandle()
+ instanceof PlaceholderKvFileHandler))
+ .mapToLong(file -> file.getKvFileHandle().getSize())
+ .sum();
+ assertThat(remoteKvCopyBytes.getCount()).isEqualTo(bytesBefore + uploadedBytes);
+ return handle;
}
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/metrics/group/TableMetricGroupTest.java b/fluss-server/src/test/java/org/apache/fluss/server/metrics/group/TableMetricGroupTest.java
index 1ddf5c9a9..ad13f63a0 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/metrics/group/TableMetricGroupTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/metrics/group/TableMetricGroupTest.java
@@ -18,16 +18,91 @@
package org.apache.fluss.server.metrics.group;
import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.metrics.Counter;
+import org.apache.fluss.metrics.MeterView;
import org.apache.fluss.metrics.MetricNames;
+import org.apache.fluss.metrics.ThreadSafeSimpleCounter;
+import org.apache.fluss.metrics.groups.AbstractMetricGroup;
+import org.apache.fluss.metrics.registry.MetricRegistry;
import org.apache.fluss.metrics.registry.NOPMetricRegistry;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
/** Test for {@link TableMetricGroup}. */
class TableMetricGroupTest {
+ @Test
+ void testRemoteKvCopyRateMetric() {
+ MetricRegistry registry = mock(MetricRegistry.class);
+ TableMetricGroup group =
+ new TableMetricGroup(
+ registry,
+ TablePath.of("db", "pk_table"),
+ true,
+ TestingMetricGroups.TABLET_SERVER_METRICS);
+ try {
+ Counter counter = group.remoteKvCopyBytes();
+ assertThat(counter).isInstanceOf(ThreadSafeSimpleCounter.class);
+ ArgumentCaptor<AbstractMetricGroup> metricGroup =
+ ArgumentCaptor.forClass(AbstractMetricGroup.class);
+ ArgumentCaptor<MeterView> meter = ArgumentCaptor.forClass(MeterView.class);
+ verify(registry)
+ .register(
+ meter.capture(),
+ eq(MetricNames.REMOTE_KV_COPY_BYTES_RATE),
+ metricGroup.capture());
+ verify(registry, never()).register(eq(counter), any(), any());
+ assertThat(metricGroup.getValue().getMetrics())
+ .containsEntry(MetricNames.REMOTE_KV_COPY_BYTES_RATE, meter.getValue())
+ .doesNotContainKey("remoteKvCopyBytes");
+ assertThat(metricGroup.getValue().getAllVariables())
+ .containsEntry("database", "db")
+ .containsEntry("table", "pk_table");
+ assertThat(metricGroup.getValue().getScopeComponents()).endsWith("kv");
+ counter.inc(120L);
+ assertThat(meter.getValue().getCount()).isEqualTo(120L);
+ meter.getValue().update();
+ assertThat(meter.getValue().getRate()).isEqualTo(2.0);
+ group.close();
+ verify(registry, never()).unregister(eq(counter), any(), any());
+ verify(registry)
+ .unregister(
+ meter.getValue(),
+ MetricNames.REMOTE_KV_COPY_BYTES_RATE,
+ metricGroup.getValue());
+ } finally {
+ group.close();
+ }
+ }
+
+ @Test
+ void testLogTableDoesNotExposeRemoteKvCopyRateMetric() {
+ MetricRegistry registry = mock(MetricRegistry.class);
+ TableMetricGroup group =
+ new TableMetricGroup(
+ registry,
+ TablePath.of("db", "log_table"),
+ false,
+ TestingMetricGroups.TABLET_SERVER_METRICS);
+ try {
+ group.remoteKvCopyBytes().inc(120L);
+ assertThat(group.remoteKvCopyBytes().getCount()).isZero();
+ verify(registry, never()).register(any(), eq("remoteKvCopyBytes"), any());
+ verify(registry, never())
+ .register(any(), eq(MetricNames.REMOTE_KV_COPY_BYTES_RATE), any());
+ } finally {
+ group.close();
+ }
+ }
+
@Test
void testKvBackpressureMetricsExposeOnlyCoreSignals() {
TableMetricGroup metricGroup =
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java
index b9e8cb7ce..50a9dc79e 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java
@@ -612,6 +612,7 @@ final class ReplicaTest extends ReplicaTestBase {
Tuple2.of("k2", new Object[] {3, "b1"}));
putRecordsToLeader(kvReplica, kvRecords);
+ long uploadedBytesBefore = kvReplica.tableMetrics().remoteKvCopyBytes().getCount();
// trigger one snapshot (task has been scheduled after becoming leader)
scheduledExecutorService.triggerAllNonPeriodicTasks();
@@ -627,6 +628,10 @@ final class ReplicaTest extends ReplicaTestBase {
Tuple2.of("k1", new Object[] {2, "b"}),
Tuple2.of("k2", new Object[] {3, "b1"})));
KvTestUtils.checkSnapshot(completedSnapshot0, expectedKeyValues, expectedLogOffset);
+ assertThat(kvReplica.tableMetrics().remoteKvCopyBytes().getCount())
+ .isEqualTo(
+ uploadedBytesBefore
+ + completedSnapshot0.getKvSnapshotHandle().getIncrementalSize());
// put some data again
kvRecords =
@@ -644,6 +649,11 @@ final class ReplicaTest extends ReplicaTestBase {
expectedKeyValues =
getKeyValuePairs(genKvRecords(Tuple2.of("k2", new Object[] {4, "bk2"})));
KvTestUtils.checkSnapshot(completedSnapshot1, expectedKeyValues, expectedLogOffset);
+ assertThat(kvReplica.tableMetrics().remoteKvCopyBytes().getCount())
+ .isEqualTo(
+ uploadedBytesBefore
+ + completedSnapshot0.getKvSnapshotHandle().getIncrementalSize()
+ + completedSnapshot1.getKvSnapshotHandle().getIncrementalSize());
// check the snapshot should be incremental, with only one newly file
KvTestUtils.checkSnapshotIncrementWithNewlyFiles(
diff --git a/website/docs/maintenance/observability/monitor-metrics.md b/website/docs/maintenance/observability/monitor-metrics.md
index 3cbc4e293..ee57382f5 100644
--- a/website/docs/maintenance/observability/monitor-metrics.md
+++ b/website/docs/maintenance/observability/monitor-metrics.md
@@ -806,8 +806,8 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM
</thead>
<tbody>
<tr>
- <th rowspan="39"><strong>tabletserver</strong></th>
- <td rowspan="20">table</td>
+ <th rowspan="41"><strong>tabletserver</strong></th>
+ <td rowspan="22">table</td>
<td>messagesInPerSecond</td>
<td>The number of messages written per second to this table.</td>
<td>Meter</td>
@@ -907,6 +907,11 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM
<td>The number of failed delete remote log requests to delete remote log after log ttl per second.</td>
<td>Meter</td>
</tr>
+ <tr>
+ <td>remoteKvCopyBytesPerSecond</td>
+ <td>The bytes of kv snapshot data uploaded to remote per second for this table. Only available for primary key tables.</td>
+ <td>Meter</td>
+ </tr>
<tr>
<td rowspan="6">table_historical</td>
<td>totalPutKvRequestsPerSecond</td>