You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by ho...@apache.org on 2026/08/03 03:43:00 UTC

(fluss) branch main updated: [flink] Remove the default exposure of client-side bucket-level metrics to reduce metrics storage pressure for existing Flink jobs. (#3828)

This is an automated email from the ASF dual-hosted git repository.

loserwang1024 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git


The following commit(s) were added to refs/heads/main by this push:
     new a5d79f7b1 [flink] Remove the default exposure of client-side bucket-level metrics to reduce metrics storage pressure for existing Flink jobs. (#3828)
a5d79f7b1 is described below

commit a5d79f7b1fbe1905a5494b856cc6268def1fa5d3
Author: Hongshun Wang <12...@users.noreply.github.com>
AuthorDate: Mon Aug 3 11:42:55 2026 +0800

    [flink] Remove the default exposure of client-side bucket-level metrics to reduce metrics storage pressure for existing Flink jobs. (#3828)
---
 .../source/metrics/FlinkSourceReaderMetrics.java   | 75 +---------------------
 .../source/reader/FlinkRecordsWithSplitIds.java    | 30 ++-------
 .../source/reader/FlinkSourceSplitReader.java      | 24 ++-----
 .../metrics/FlinkSourceReaderMetricsTest.java      | 63 +++---------------
 4 files changed, 24 insertions(+), 168 deletions(-)

diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/metrics/FlinkSourceReaderMetrics.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/metrics/FlinkSourceReaderMetrics.java
index dc98ada82..400f1969f 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/metrics/FlinkSourceReaderMetrics.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/metrics/FlinkSourceReaderMetrics.java
@@ -18,60 +18,23 @@
 package org.apache.fluss.flink.source.metrics;
 
 import org.apache.fluss.flink.source.reader.FlinkSourceReader;
-import org.apache.fluss.metadata.TableBucket;
 
-import org.apache.flink.metrics.MetricGroup;
 import org.apache.flink.metrics.groups.SourceReaderMetricGroup;
 import org.apache.flink.runtime.metrics.MetricNames;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * A collection class for handling metrics in {@link FlinkSourceReader} of Fluss.
- *
- * <p>All metrics of the source reader are registered under group "fluss.reader", which is a child
- * group of {@link org.apache.flink.metrics.groups.OperatorMetricGroup}. Metrics related to a
- * specific table bucket will be registered in the group:
- *
- * <p>"fluss.reader.bucket.{bucket_id}" for non-partitioned bucket or
- * "fluss.reader.partition.{partition_id}.bucket.{bucket_id}" for partitioned bucket.
- *
- * <p>For example, current consuming offset of table "my-table" and bucket 1 will be reported in
- * metric: "{some_parent_groups}.operator.fluss.reader.table.my-table.bucket.1.currentOffset"
- */
+/** A collection class for handling metrics in {@link FlinkSourceReader} of Fluss. */
 public class FlinkSourceReaderMetrics {
 
-    private static final Logger LOG = LoggerFactory.getLogger(FlinkSourceReaderMetrics.class);
-
-    // Constants
-    public static final String FLUSS_METRIC_GROUP = "fluss";
-    public static final String READER_METRIC_GROUP = "reader";
-    public static final String PARTITION_GROUP = "partition";
-    public static final String BUCKET_GROUP = "bucket";
-    public static final String CURRENT_OFFSET_METRIC_GAUGE = "currentOffset";
-
-    public static final long INITIAL_OFFSET = -1;
     public static final long UNINITIALIZED = -1;
 
     // Source reader metric group
     private final SourceReaderMetricGroup sourceReaderMetricGroup;
 
-    // Metric group for registering Fluss specific reader metrics
-    private final MetricGroup flussSourceReaderMetricGroup;
-
-    // Map for tracking current consuming offsets
-    private final Map<TableBucket, Long> offsets = new HashMap<>();
-
     // For currentFetchEventTimeLag metric
     private volatile long currentFetchEventTimeLag = UNINITIALIZED;
 
     public FlinkSourceReaderMetrics(SourceReaderMetricGroup sourceReaderMetricGroup) {
         this.sourceReaderMetricGroup = sourceReaderMetricGroup;
-        this.flussSourceReaderMetricGroup =
-                sourceReaderMetricGroup.addGroup(FLUSS_METRIC_GROUP).addGroup(READER_METRIC_GROUP);
     }
 
     public void reportRecordEventTime(long lag) {
@@ -87,42 +50,6 @@ public class FlinkSourceReaderMetrics {
         currentFetchEventTimeLag = lag;
     }
 
-    public void registerTableBucket(TableBucket tableBucket) {
-        offsets.put(tableBucket, INITIAL_OFFSET);
-        registerOffsetMetricsForTableBucket(tableBucket);
-    }
-
-    /**
-     * Update current consuming offset of the given {@link TableBucket}.
-     *
-     * @param tb Updating table bucket
-     * @param offset Current consuming offset
-     */
-    public void recordCurrentOffset(TableBucket tb, long offset) {
-        checkTableBucketTracked(tb);
-        offsets.put(tb, offset);
-    }
-
-    // -------- Helper functions --------
-    private void registerOffsetMetricsForTableBucket(TableBucket tableBucket) {
-        final MetricGroup metricGroup =
-                tableBucket.getPartitionId() == null
-                        ? this.flussSourceReaderMetricGroup
-                        : this.flussSourceReaderMetricGroup.addGroup(
-                                PARTITION_GROUP, String.valueOf(tableBucket.getPartitionId()));
-        final MetricGroup bucketGroup =
-                metricGroup.addGroup(BUCKET_GROUP, String.valueOf(tableBucket.getBucket()));
-        bucketGroup.gauge(
-                CURRENT_OFFSET_METRIC_GAUGE,
-                () -> offsets.getOrDefault(tableBucket, INITIAL_OFFSET));
-    }
-
-    private void checkTableBucketTracked(TableBucket tableBucket) {
-        if (!offsets.containsKey(tableBucket)) {
-            LOG.warn("Offset metrics of TableBucket {} is not tracked", tableBucket);
-        }
-    }
-
     public SourceReaderMetricGroup getSourceReaderMetricGroup() {
         return sourceReaderMetricGroup;
     }
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkRecordsWithSplitIds.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkRecordsWithSplitIds.java
index cf5c60a67..3b7babe1a 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkRecordsWithSplitIds.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkRecordsWithSplitIds.java
@@ -17,7 +17,6 @@
 
 package org.apache.fluss.flink.source.reader;
 
-import org.apache.fluss.flink.source.metrics.FlinkSourceReaderMetrics;
 import org.apache.fluss.metadata.TableBucket;
 import org.apache.fluss.utils.CloseableIterator;
 
@@ -50,54 +49,43 @@ public class FlinkRecordsWithSplitIds implements RecordsWithSplitIds<RecordAndPo
     /** The table buckets of the split in splitIterator. */
     private final Iterator<TableBucket> tableBucketIterator;
 
-    private final FlinkSourceReaderMetrics flinkSourceReaderMetrics;
-
     // the closable iterator for the records in the current split
     private @Nullable CloseableIterator<RecordAndPos> currentRecordIterator;
     private @Nullable TableBucket currentTableBucket;
     private @Nullable Long currentSplitStoppingOffset;
 
-    public static FlinkRecordsWithSplitIds emptyRecords(
-            FlinkSourceReaderMetrics flinkSourceReaderMetrics) {
-        return new FlinkRecordsWithSplitIds(Collections.emptySet(), flinkSourceReaderMetrics);
+    public static FlinkRecordsWithSplitIds emptyRecords() {
+        return new FlinkRecordsWithSplitIds(Collections.emptySet());
     }
 
     // only for single split
     public FlinkRecordsWithSplitIds(
-            String split,
-            TableBucket tableBucket,
-            CloseableIterator<RecordAndPos> records,
-            FlinkSourceReaderMetrics flinkSourceReaderMetrics) {
+            String split, TableBucket tableBucket, CloseableIterator<RecordAndPos> records) {
         this(
                 Collections.singletonMap(split, records),
                 Collections.singleton(split).iterator(),
                 Collections.singleton(tableBucket).iterator(),
-                new HashSet<>(),
-                flinkSourceReaderMetrics);
+                new HashSet<>());
     }
 
     // no any splits, just used to mark splits finished
-    public FlinkRecordsWithSplitIds(
-            Set<String> finishedSplits, FlinkSourceReaderMetrics flinkSourceReaderMetrics) {
+    public FlinkRecordsWithSplitIds(Set<String> finishedSplits) {
         this(
                 Collections.emptyMap(),
                 Collections.emptyIterator(),
                 Collections.emptyIterator(),
-                finishedSplits,
-                flinkSourceReaderMetrics);
+                finishedSplits);
     }
 
     public FlinkRecordsWithSplitIds(
             Map<String, CloseableIterator<RecordAndPos>> splitRecords,
             Iterator<String> splitIterator,
             Iterator<TableBucket> tableBucketIterator,
-            Set<String> finishedSplits,
-            FlinkSourceReaderMetrics flinkSourceReaderMetrics) {
+            Set<String> finishedSplits) {
         this.splitRecords = splitRecords;
         this.splitIterator = splitIterator;
         this.tableBucketIterator = tableBucketIterator;
         this.finishedSplits = finishedSplits;
-        this.flinkSourceReaderMetrics = flinkSourceReaderMetrics;
     }
 
     public void setTableBucketStoppingOffset(TableBucket tableBucket, long stoppingOffset) {
@@ -138,10 +126,6 @@ public class FlinkRecordsWithSplitIds implements RecordsWithSplitIds<RecordAndPo
                 return null;
             }
 
-            if (offset >= 0) {
-                flinkSourceReaderMetrics.recordCurrentOffset(currentTableBucket, offset);
-            }
-
             return recordAndPos;
         }
         return null;
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
index 6c9648284..50e214396 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
@@ -152,8 +152,7 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
     public RecordsWithSplitIds<RecordAndPos> fetch() throws IOException {
         if (!removedSplits.isEmpty()) {
             FlinkRecordsWithSplitIds records =
-                    new FlinkRecordsWithSplitIds(
-                            new HashSet<>(removedSplits), flinkSourceReaderMetrics);
+                    new FlinkRecordsWithSplitIds(new HashSet<>(removedSplits));
             removedSplits.clear();
             return records;
         }
@@ -170,14 +169,13 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
             // may need to finish empty log splits
             if (!emptyLogSplits.isEmpty()) {
                 FlinkRecordsWithSplitIds records =
-                        new FlinkRecordsWithSplitIds(
-                                new HashSet<>(emptyLogSplits), flinkSourceReaderMetrics);
+                        new FlinkRecordsWithSplitIds(new HashSet<>(emptyLogSplits));
                 emptyLogSplits.clear();
                 return records;
             } else {
                 // if not subscribe any buckets, just return empty records
                 if (subscribedBuckets.isEmpty()) {
-                    return FlinkRecordsWithSplitIds.emptyRecords(flinkSourceReaderMetrics);
+                    return FlinkRecordsWithSplitIds.emptyRecords();
                 }
                 ScanRecords scanRecords = logScanner.poll(POLL_TIMEOUT);
                 return forLogRecords(scanRecords);
@@ -317,8 +315,6 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
                     "Subscribe to read log for split {} from offset {}.",
                     split.splitId(),
                     startingOffset);
-            // Track the new bucket in metrics and internal state.
-            flinkSourceReaderMetrics.registerTableBucket(tableBucket);
             subscribedBuckets.put(tableBucket, split.splitId());
         }
     }
@@ -513,11 +509,7 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
 
         FlinkRecordsWithSplitIds recordsWithSplitIds =
                 new FlinkRecordsWithSplitIds(
-                        splitRecords,
-                        splitIterator,
-                        tableScanBuckets.iterator(),
-                        finishedSplits,
-                        flinkSourceReaderMetrics);
+                        splitRecords, splitIterator, tableScanBuckets.iterator(), finishedSplits);
         stoppingOffsets.forEach(recordsWithSplitIds::setTableBucketStoppingOffset);
         return recordsWithSplitIds;
     }
@@ -547,10 +539,7 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
             final SourceSplitBase snapshotSplit,
             final CloseableIterator<RecordAndPos> recordsForSplit) {
         return new FlinkRecordsWithSplitIds(
-                snapshotSplit.splitId(),
-                snapshotSplit.getTableBucket(),
-                recordsForSplit,
-                flinkSourceReaderMetrics);
+                snapshotSplit.splitId(), snapshotSplit.getTableBucket(), recordsForSplit);
     }
 
     private long getStoppingOffset(TableBucket tableBucket) {
@@ -570,8 +559,7 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
                         // since it remains log to read
                         ? Collections.emptySet()
                         : Collections.singleton(currentBoundedSplit.splitId());
-        final FlinkRecordsWithSplitIds finishRecords =
-                new FlinkRecordsWithSplitIds(finishedSplits, flinkSourceReaderMetrics);
+        final FlinkRecordsWithSplitIds finishRecords = new FlinkRecordsWithSplitIds(finishedSplits);
         closeCurrentBoundedSplit();
         return finishRecords;
     }
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/metrics/FlinkSourceReaderMetricsTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/metrics/FlinkSourceReaderMetricsTest.java
index 255a5b680..f7bb7f2fc 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/metrics/FlinkSourceReaderMetricsTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/metrics/FlinkSourceReaderMetricsTest.java
@@ -17,79 +17,36 @@
 
 package org.apache.fluss.flink.source.metrics;
 
-import org.apache.fluss.metadata.TableBucket;
-
 import org.apache.flink.metrics.Gauge;
 import org.apache.flink.metrics.testutils.MetricListener;
+import org.apache.flink.runtime.metrics.MetricNames;
 import org.apache.flink.runtime.metrics.groups.InternalSourceReaderMetricGroup;
 import org.junit.jupiter.api.Test;
 
 import java.util.Optional;
 
-import static org.apache.fluss.flink.source.metrics.FlinkSourceReaderMetrics.BUCKET_GROUP;
-import static org.apache.fluss.flink.source.metrics.FlinkSourceReaderMetrics.FLUSS_METRIC_GROUP;
-import static org.apache.fluss.flink.source.metrics.FlinkSourceReaderMetrics.PARTITION_GROUP;
-import static org.apache.fluss.flink.source.metrics.FlinkSourceReaderMetrics.READER_METRIC_GROUP;
 import static org.assertj.core.api.Assertions.assertThat;
 
 /** Unit test for {@link org.apache.fluss.flink.source.metrics.FlinkSourceReaderMetrics}. */
 class FlinkSourceReaderMetricsTest {
 
     @Test
-    void testCurrentOffsetTracking() {
+    void testCurrentFetchEventTimeLag() {
         MetricListener metricListener = new MetricListener();
 
-        final TableBucket t0 = new TableBucket(0, 0L, 1);
-        final TableBucket t1 = new TableBucket(0, 0L, 2);
-        final TableBucket t2 = new TableBucket(0, null, 1);
-        final TableBucket t3 = new TableBucket(0, null, 2);
-
         final FlinkSourceReaderMetrics flinkSourceReaderMetrics =
                 new FlinkSourceReaderMetrics(
                         InternalSourceReaderMetricGroup.mock(metricListener.getMetricGroup()));
 
-        flinkSourceReaderMetrics.registerTableBucket(t0);
-        flinkSourceReaderMetrics.registerTableBucket(t1);
-        flinkSourceReaderMetrics.registerTableBucket(t2);
-        flinkSourceReaderMetrics.registerTableBucket(t3);
-
-        flinkSourceReaderMetrics.recordCurrentOffset(t0, 15213L);
-        flinkSourceReaderMetrics.recordCurrentOffset(t1, 18213L);
-        flinkSourceReaderMetrics.recordCurrentOffset(t2, 18613L);
-        flinkSourceReaderMetrics.recordCurrentOffset(t3, 15513L);
-
-        assertCurrentOffset(t0, 15213L, metricListener);
-        assertCurrentOffset(t1, 18213L, metricListener);
-        assertCurrentOffset(t2, 18613L, metricListener);
-        assertCurrentOffset(t3, 15513L, metricListener);
-    }
-
-    // ----------- Assertions --------------
+        assertThat(metricListener.getGauge(MetricNames.CURRENT_FETCH_EVENT_TIME_LAG)).isEmpty();
 
-    private void assertCurrentOffset(
-            TableBucket tb, long expectedOffset, MetricListener metricListener) {
-        final Optional<Gauge<Long>> currentOffsetGauge;
-        if (tb.getPartitionId() == null) {
-            currentOffsetGauge =
-                    metricListener.getGauge(
-                            FLUSS_METRIC_GROUP,
-                            READER_METRIC_GROUP,
-                            BUCKET_GROUP,
-                            String.valueOf(tb.getBucket()),
-                            FlinkSourceReaderMetrics.CURRENT_OFFSET_METRIC_GAUGE);
-        } else {
-            currentOffsetGauge =
-                    metricListener.getGauge(
-                            FLUSS_METRIC_GROUP,
-                            READER_METRIC_GROUP,
-                            PARTITION_GROUP,
-                            String.valueOf(tb.getPartitionId()),
-                            BUCKET_GROUP,
-                            String.valueOf(tb.getBucket()),
-                            FlinkSourceReaderMetrics.CURRENT_OFFSET_METRIC_GAUGE);
-        }
+        flinkSourceReaderMetrics.reportRecordEventTime(15213L);
+        Optional<Gauge<Long>> currentFetchEventTimeLag =
+                metricListener.getGauge(MetricNames.CURRENT_FETCH_EVENT_TIME_LAG);
+        assertThat(currentFetchEventTimeLag).isPresent();
+        assertThat((long) currentFetchEventTimeLag.get().getValue()).isEqualTo(15213L);
 
-        assertThat(currentOffsetGauge).isPresent();
-        assertThat((long) currentOffsetGauge.get().getValue()).isEqualTo(expectedOffset);
+        flinkSourceReaderMetrics.reportRecordEventTime(18213L);
+        assertThat((long) currentFetchEventTimeLag.get().getValue()).isEqualTo(18213L);
     }
 }