You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by ho...@apache.org on 2026/08/03 03:43:00 UTC
(fluss) branch main updated: [flink] Remove the default exposure of client-side bucket-level metrics to reduce metrics storage pressure for existing Flink jobs. (#3828)
This is an automated email from the ASF dual-hosted git repository.
loserwang1024 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
The following commit(s) were added to refs/heads/main by this push:
new a5d79f7b1 [flink] Remove the default exposure of client-side bucket-level metrics to reduce metrics storage pressure for existing Flink jobs. (#3828)
a5d79f7b1 is described below
commit a5d79f7b1fbe1905a5494b856cc6268def1fa5d3
Author: Hongshun Wang <12...@users.noreply.github.com>
AuthorDate: Mon Aug 3 11:42:55 2026 +0800
[flink] Remove the default exposure of client-side bucket-level metrics to reduce metrics storage pressure for existing Flink jobs. (#3828)
---
.../source/metrics/FlinkSourceReaderMetrics.java | 75 +---------------------
.../source/reader/FlinkRecordsWithSplitIds.java | 30 ++-------
.../source/reader/FlinkSourceSplitReader.java | 24 ++-----
.../metrics/FlinkSourceReaderMetricsTest.java | 63 +++---------------
4 files changed, 24 insertions(+), 168 deletions(-)
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/metrics/FlinkSourceReaderMetrics.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/metrics/FlinkSourceReaderMetrics.java
index dc98ada82..400f1969f 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/metrics/FlinkSourceReaderMetrics.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/metrics/FlinkSourceReaderMetrics.java
@@ -18,60 +18,23 @@
package org.apache.fluss.flink.source.metrics;
import org.apache.fluss.flink.source.reader.FlinkSourceReader;
-import org.apache.fluss.metadata.TableBucket;
-import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.metrics.groups.SourceReaderMetricGroup;
import org.apache.flink.runtime.metrics.MetricNames;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * A collection class for handling metrics in {@link FlinkSourceReader} of Fluss.
- *
- * <p>All metrics of the source reader are registered under group "fluss.reader", which is a child
- * group of {@link org.apache.flink.metrics.groups.OperatorMetricGroup}. Metrics related to a
- * specific table bucket will be registered in the group:
- *
- * <p>"fluss.reader.bucket.{bucket_id}" for non-partitioned bucket or
- * "fluss.reader.partition.{partition_id}.bucket.{bucket_id}" for partitioned bucket.
- *
- * <p>For example, current consuming offset of table "my-table" and bucket 1 will be reported in
- * metric: "{some_parent_groups}.operator.fluss.reader.table.my-table.bucket.1.currentOffset"
- */
+/** A collection class for handling metrics in {@link FlinkSourceReader} of Fluss. */
public class FlinkSourceReaderMetrics {
- private static final Logger LOG = LoggerFactory.getLogger(FlinkSourceReaderMetrics.class);
-
- // Constants
- public static final String FLUSS_METRIC_GROUP = "fluss";
- public static final String READER_METRIC_GROUP = "reader";
- public static final String PARTITION_GROUP = "partition";
- public static final String BUCKET_GROUP = "bucket";
- public static final String CURRENT_OFFSET_METRIC_GAUGE = "currentOffset";
-
- public static final long INITIAL_OFFSET = -1;
public static final long UNINITIALIZED = -1;
// Source reader metric group
private final SourceReaderMetricGroup sourceReaderMetricGroup;
- // Metric group for registering Fluss specific reader metrics
- private final MetricGroup flussSourceReaderMetricGroup;
-
- // Map for tracking current consuming offsets
- private final Map<TableBucket, Long> offsets = new HashMap<>();
-
// For currentFetchEventTimeLag metric
private volatile long currentFetchEventTimeLag = UNINITIALIZED;
public FlinkSourceReaderMetrics(SourceReaderMetricGroup sourceReaderMetricGroup) {
this.sourceReaderMetricGroup = sourceReaderMetricGroup;
- this.flussSourceReaderMetricGroup =
- sourceReaderMetricGroup.addGroup(FLUSS_METRIC_GROUP).addGroup(READER_METRIC_GROUP);
}
public void reportRecordEventTime(long lag) {
@@ -87,42 +50,6 @@ public class FlinkSourceReaderMetrics {
currentFetchEventTimeLag = lag;
}
- public void registerTableBucket(TableBucket tableBucket) {
- offsets.put(tableBucket, INITIAL_OFFSET);
- registerOffsetMetricsForTableBucket(tableBucket);
- }
-
- /**
- * Update current consuming offset of the given {@link TableBucket}.
- *
- * @param tb Updating table bucket
- * @param offset Current consuming offset
- */
- public void recordCurrentOffset(TableBucket tb, long offset) {
- checkTableBucketTracked(tb);
- offsets.put(tb, offset);
- }
-
- // -------- Helper functions --------
- private void registerOffsetMetricsForTableBucket(TableBucket tableBucket) {
- final MetricGroup metricGroup =
- tableBucket.getPartitionId() == null
- ? this.flussSourceReaderMetricGroup
- : this.flussSourceReaderMetricGroup.addGroup(
- PARTITION_GROUP, String.valueOf(tableBucket.getPartitionId()));
- final MetricGroup bucketGroup =
- metricGroup.addGroup(BUCKET_GROUP, String.valueOf(tableBucket.getBucket()));
- bucketGroup.gauge(
- CURRENT_OFFSET_METRIC_GAUGE,
- () -> offsets.getOrDefault(tableBucket, INITIAL_OFFSET));
- }
-
- private void checkTableBucketTracked(TableBucket tableBucket) {
- if (!offsets.containsKey(tableBucket)) {
- LOG.warn("Offset metrics of TableBucket {} is not tracked", tableBucket);
- }
- }
-
public SourceReaderMetricGroup getSourceReaderMetricGroup() {
return sourceReaderMetricGroup;
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkRecordsWithSplitIds.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkRecordsWithSplitIds.java
index cf5c60a67..3b7babe1a 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkRecordsWithSplitIds.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkRecordsWithSplitIds.java
@@ -17,7 +17,6 @@
package org.apache.fluss.flink.source.reader;
-import org.apache.fluss.flink.source.metrics.FlinkSourceReaderMetrics;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.utils.CloseableIterator;
@@ -50,54 +49,43 @@ public class FlinkRecordsWithSplitIds implements RecordsWithSplitIds<RecordAndPo
/** The table buckets of the split in splitIterator. */
private final Iterator<TableBucket> tableBucketIterator;
- private final FlinkSourceReaderMetrics flinkSourceReaderMetrics;
-
// the closable iterator for the records in the current split
private @Nullable CloseableIterator<RecordAndPos> currentRecordIterator;
private @Nullable TableBucket currentTableBucket;
private @Nullable Long currentSplitStoppingOffset;
- public static FlinkRecordsWithSplitIds emptyRecords(
- FlinkSourceReaderMetrics flinkSourceReaderMetrics) {
- return new FlinkRecordsWithSplitIds(Collections.emptySet(), flinkSourceReaderMetrics);
+ public static FlinkRecordsWithSplitIds emptyRecords() {
+ return new FlinkRecordsWithSplitIds(Collections.emptySet());
}
// only for single split
public FlinkRecordsWithSplitIds(
- String split,
- TableBucket tableBucket,
- CloseableIterator<RecordAndPos> records,
- FlinkSourceReaderMetrics flinkSourceReaderMetrics) {
+ String split, TableBucket tableBucket, CloseableIterator<RecordAndPos> records) {
this(
Collections.singletonMap(split, records),
Collections.singleton(split).iterator(),
Collections.singleton(tableBucket).iterator(),
- new HashSet<>(),
- flinkSourceReaderMetrics);
+ new HashSet<>());
}
// no any splits, just used to mark splits finished
- public FlinkRecordsWithSplitIds(
- Set<String> finishedSplits, FlinkSourceReaderMetrics flinkSourceReaderMetrics) {
+ public FlinkRecordsWithSplitIds(Set<String> finishedSplits) {
this(
Collections.emptyMap(),
Collections.emptyIterator(),
Collections.emptyIterator(),
- finishedSplits,
- flinkSourceReaderMetrics);
+ finishedSplits);
}
public FlinkRecordsWithSplitIds(
Map<String, CloseableIterator<RecordAndPos>> splitRecords,
Iterator<String> splitIterator,
Iterator<TableBucket> tableBucketIterator,
- Set<String> finishedSplits,
- FlinkSourceReaderMetrics flinkSourceReaderMetrics) {
+ Set<String> finishedSplits) {
this.splitRecords = splitRecords;
this.splitIterator = splitIterator;
this.tableBucketIterator = tableBucketIterator;
this.finishedSplits = finishedSplits;
- this.flinkSourceReaderMetrics = flinkSourceReaderMetrics;
}
public void setTableBucketStoppingOffset(TableBucket tableBucket, long stoppingOffset) {
@@ -138,10 +126,6 @@ public class FlinkRecordsWithSplitIds implements RecordsWithSplitIds<RecordAndPo
return null;
}
- if (offset >= 0) {
- flinkSourceReaderMetrics.recordCurrentOffset(currentTableBucket, offset);
- }
-
return recordAndPos;
}
return null;
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
index 6c9648284..50e214396 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
@@ -152,8 +152,7 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
public RecordsWithSplitIds<RecordAndPos> fetch() throws IOException {
if (!removedSplits.isEmpty()) {
FlinkRecordsWithSplitIds records =
- new FlinkRecordsWithSplitIds(
- new HashSet<>(removedSplits), flinkSourceReaderMetrics);
+ new FlinkRecordsWithSplitIds(new HashSet<>(removedSplits));
removedSplits.clear();
return records;
}
@@ -170,14 +169,13 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
// may need to finish empty log splits
if (!emptyLogSplits.isEmpty()) {
FlinkRecordsWithSplitIds records =
- new FlinkRecordsWithSplitIds(
- new HashSet<>(emptyLogSplits), flinkSourceReaderMetrics);
+ new FlinkRecordsWithSplitIds(new HashSet<>(emptyLogSplits));
emptyLogSplits.clear();
return records;
} else {
// if not subscribe any buckets, just return empty records
if (subscribedBuckets.isEmpty()) {
- return FlinkRecordsWithSplitIds.emptyRecords(flinkSourceReaderMetrics);
+ return FlinkRecordsWithSplitIds.emptyRecords();
}
ScanRecords scanRecords = logScanner.poll(POLL_TIMEOUT);
return forLogRecords(scanRecords);
@@ -317,8 +315,6 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
"Subscribe to read log for split {} from offset {}.",
split.splitId(),
startingOffset);
- // Track the new bucket in metrics and internal state.
- flinkSourceReaderMetrics.registerTableBucket(tableBucket);
subscribedBuckets.put(tableBucket, split.splitId());
}
}
@@ -513,11 +509,7 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
FlinkRecordsWithSplitIds recordsWithSplitIds =
new FlinkRecordsWithSplitIds(
- splitRecords,
- splitIterator,
- tableScanBuckets.iterator(),
- finishedSplits,
- flinkSourceReaderMetrics);
+ splitRecords, splitIterator, tableScanBuckets.iterator(), finishedSplits);
stoppingOffsets.forEach(recordsWithSplitIds::setTableBucketStoppingOffset);
return recordsWithSplitIds;
}
@@ -547,10 +539,7 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
final SourceSplitBase snapshotSplit,
final CloseableIterator<RecordAndPos> recordsForSplit) {
return new FlinkRecordsWithSplitIds(
- snapshotSplit.splitId(),
- snapshotSplit.getTableBucket(),
- recordsForSplit,
- flinkSourceReaderMetrics);
+ snapshotSplit.splitId(), snapshotSplit.getTableBucket(), recordsForSplit);
}
private long getStoppingOffset(TableBucket tableBucket) {
@@ -570,8 +559,7 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
// since it remains log to read
? Collections.emptySet()
: Collections.singleton(currentBoundedSplit.splitId());
- final FlinkRecordsWithSplitIds finishRecords =
- new FlinkRecordsWithSplitIds(finishedSplits, flinkSourceReaderMetrics);
+ final FlinkRecordsWithSplitIds finishRecords = new FlinkRecordsWithSplitIds(finishedSplits);
closeCurrentBoundedSplit();
return finishRecords;
}
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/metrics/FlinkSourceReaderMetricsTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/metrics/FlinkSourceReaderMetricsTest.java
index 255a5b680..f7bb7f2fc 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/metrics/FlinkSourceReaderMetricsTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/metrics/FlinkSourceReaderMetricsTest.java
@@ -17,79 +17,36 @@
package org.apache.fluss.flink.source.metrics;
-import org.apache.fluss.metadata.TableBucket;
-
import org.apache.flink.metrics.Gauge;
import org.apache.flink.metrics.testutils.MetricListener;
+import org.apache.flink.runtime.metrics.MetricNames;
import org.apache.flink.runtime.metrics.groups.InternalSourceReaderMetricGroup;
import org.junit.jupiter.api.Test;
import java.util.Optional;
-import static org.apache.fluss.flink.source.metrics.FlinkSourceReaderMetrics.BUCKET_GROUP;
-import static org.apache.fluss.flink.source.metrics.FlinkSourceReaderMetrics.FLUSS_METRIC_GROUP;
-import static org.apache.fluss.flink.source.metrics.FlinkSourceReaderMetrics.PARTITION_GROUP;
-import static org.apache.fluss.flink.source.metrics.FlinkSourceReaderMetrics.READER_METRIC_GROUP;
import static org.assertj.core.api.Assertions.assertThat;
/** Unit test for {@link org.apache.fluss.flink.source.metrics.FlinkSourceReaderMetrics}. */
class FlinkSourceReaderMetricsTest {
@Test
- void testCurrentOffsetTracking() {
+ void testCurrentFetchEventTimeLag() {
MetricListener metricListener = new MetricListener();
- final TableBucket t0 = new TableBucket(0, 0L, 1);
- final TableBucket t1 = new TableBucket(0, 0L, 2);
- final TableBucket t2 = new TableBucket(0, null, 1);
- final TableBucket t3 = new TableBucket(0, null, 2);
-
final FlinkSourceReaderMetrics flinkSourceReaderMetrics =
new FlinkSourceReaderMetrics(
InternalSourceReaderMetricGroup.mock(metricListener.getMetricGroup()));
- flinkSourceReaderMetrics.registerTableBucket(t0);
- flinkSourceReaderMetrics.registerTableBucket(t1);
- flinkSourceReaderMetrics.registerTableBucket(t2);
- flinkSourceReaderMetrics.registerTableBucket(t3);
-
- flinkSourceReaderMetrics.recordCurrentOffset(t0, 15213L);
- flinkSourceReaderMetrics.recordCurrentOffset(t1, 18213L);
- flinkSourceReaderMetrics.recordCurrentOffset(t2, 18613L);
- flinkSourceReaderMetrics.recordCurrentOffset(t3, 15513L);
-
- assertCurrentOffset(t0, 15213L, metricListener);
- assertCurrentOffset(t1, 18213L, metricListener);
- assertCurrentOffset(t2, 18613L, metricListener);
- assertCurrentOffset(t3, 15513L, metricListener);
- }
-
- // ----------- Assertions --------------
+ assertThat(metricListener.getGauge(MetricNames.CURRENT_FETCH_EVENT_TIME_LAG)).isEmpty();
- private void assertCurrentOffset(
- TableBucket tb, long expectedOffset, MetricListener metricListener) {
- final Optional<Gauge<Long>> currentOffsetGauge;
- if (tb.getPartitionId() == null) {
- currentOffsetGauge =
- metricListener.getGauge(
- FLUSS_METRIC_GROUP,
- READER_METRIC_GROUP,
- BUCKET_GROUP,
- String.valueOf(tb.getBucket()),
- FlinkSourceReaderMetrics.CURRENT_OFFSET_METRIC_GAUGE);
- } else {
- currentOffsetGauge =
- metricListener.getGauge(
- FLUSS_METRIC_GROUP,
- READER_METRIC_GROUP,
- PARTITION_GROUP,
- String.valueOf(tb.getPartitionId()),
- BUCKET_GROUP,
- String.valueOf(tb.getBucket()),
- FlinkSourceReaderMetrics.CURRENT_OFFSET_METRIC_GAUGE);
- }
+ flinkSourceReaderMetrics.reportRecordEventTime(15213L);
+ Optional<Gauge<Long>> currentFetchEventTimeLag =
+ metricListener.getGauge(MetricNames.CURRENT_FETCH_EVENT_TIME_LAG);
+ assertThat(currentFetchEventTimeLag).isPresent();
+ assertThat((long) currentFetchEventTimeLag.get().getValue()).isEqualTo(15213L);
- assertThat(currentOffsetGauge).isPresent();
- assertThat((long) currentOffsetGauge.get().getValue()).isEqualTo(expectedOffset);
+ flinkSourceReaderMetrics.reportRecordEventTime(18213L);
+ assertThat((long) currentFetchEventTimeLag.get().getValue()).isEqualTo(18213L);
}
}