You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/12/26 16:24:00 UTC

(incubator-paimon) branch master updated: [flink] Remove nullable in FileStoreSourceReader

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new f7bc122e3 [flink] Remove nullable in FileStoreSourceReader
f7bc122e3 is described below

commit f7bc122e35e561855c032c3a4efc317870b644e0
Author: Jingsong <ji...@gmail.com>
AuthorDate: Wed Dec 27 00:23:52 2023 +0800

    [flink] Remove nullable in FileStoreSourceReader
---
 .../paimon/flink/source/FileStoreSourceReader.java | 29 +++++-----
 .../flink/source/FileStoreSourceSplitReader.java   | 10 ++--
 .../flink/source/FlinkRecordsWithSplitIds.java     |  8 ++-
 .../apache/paimon/flink/source/FlinkSource.java    |  2 +-
 .../align/AlignedContinuousFileStoreSource.java    |  4 +-
 .../flink/source/align/AlignedSourceReader.java    |  8 +--
 .../flink/source/FileStoreSourceReaderTest.java    | 62 +++++++++++++++++++++-
 .../source/FileStoreSourceSplitReaderTest.java     |  6 ++-
 .../source/align/AlignedSourceReaderTest.java      |  8 +--
 9 files changed, 98 insertions(+), 39 deletions(-)

diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceReader.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceReader.java
index 31882406c..58e90087b 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceReader.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceReader.java
@@ -41,23 +41,23 @@ public class FileStoreSourceReader
         extends SingleThreadMultiplexSourceReaderBase<
                 RecordIterator<RowData>, RowData, FileStoreSourceSplit, FileStoreSourceSplitState> {
 
-    @Nullable private final IOManager ioManager;
+    private final IOManager ioManager;
+
     private long lastConsumeSnapshotId = Long.MIN_VALUE;
 
     public FileStoreSourceReader(
             SourceReaderContext readerContext,
             TableRead tableRead,
-            @Nullable IOManager ioManager,
-            @Nullable Long limit,
-            @Nullable FileStoreSourceReaderMetrics sourceReaderMetrics) {
+            FileStoreSourceReaderMetrics metrics,
+            IOManager ioManager,
+            @Nullable Long limit) {
         // limiter is created in SourceReader, it can be shared in all split readers
         super(
                 () ->
                         new FileStoreSourceSplitReader(
-                                tableRead, RecordLimiter.create(limit), sourceReaderMetrics),
+                                tableRead, RecordLimiter.create(limit), metrics),
                 (element, output, state) ->
-                        FlinkRecordsWithSplitIds.emitRecord(
-                                element, output, state, sourceReaderMetrics),
+                        FlinkRecordsWithSplitIds.emitRecord(element, output, state, metrics),
                 readerContext.getConfiguration(),
                 readerContext);
         this.ioManager = ioManager;
@@ -66,19 +66,18 @@ public class FileStoreSourceReader
     public FileStoreSourceReader(
             SourceReaderContext readerContext,
             TableRead tableRead,
-            @Nullable IOManager ioManager,
+            FileStoreSourceReaderMetrics metrics,
+            IOManager ioManager,
             @Nullable Long limit,
             FutureCompletingBlockingQueue<RecordsWithSplitIds<RecordIterator<RowData>>>
-                    elementsQueue,
-            @Nullable FileStoreSourceReaderMetrics sourceReaderMetrics) {
+                    elementsQueue) {
         super(
                 elementsQueue,
                 () ->
                         new FileStoreSourceSplitReader(
-                                tableRead, RecordLimiter.create(limit), sourceReaderMetrics),
+                                tableRead, RecordLimiter.create(limit), metrics),
                 (element, output, state) ->
-                        FlinkRecordsWithSplitIds.emitRecord(
-                                element, output, state, sourceReaderMetrics),
+                        FlinkRecordsWithSplitIds.emitRecord(element, output, state, metrics),
                 readerContext.getConfiguration(),
                 readerContext);
         this.ioManager = ioManager;
@@ -130,8 +129,6 @@ public class FileStoreSourceReader
     @Override
     public void close() throws Exception {
         super.close();
-        if (ioManager != null) {
-            ioManager.close();
-        }
+        ioManager.close();
     }
 }
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceSplitReader.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceSplitReader.java
index 7260650be..c70c339d7 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceSplitReader.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceSplitReader.java
@@ -65,19 +65,19 @@ public class FileStoreSourceSplitReader
     private RecordIterator<InternalRow> currentFirstBatch;
 
     private boolean paused;
-    private final FileStoreSourceReaderMetrics sourceReaderMetrics;
+    private final FileStoreSourceReaderMetrics metrics;
 
     public FileStoreSourceSplitReader(
             TableRead tableRead,
             @Nullable RecordLimiter limiter,
-            @Nullable FileStoreSourceReaderMetrics sourceReaderMetrics) {
+            FileStoreSourceReaderMetrics metrics) {
         this.tableRead = tableRead;
         this.limiter = limiter;
         this.splits = new LinkedList<>();
         this.pool = new Pool<>(1);
         this.pool.add(new FileStoreRecordIterator());
         this.paused = false;
-        this.sourceReaderMetrics = sourceReaderMetrics;
+        this.metrics = metrics;
     }
 
     @Override
@@ -178,12 +178,12 @@ public class FileStoreSourceSplitReader
         }
 
         // update metric when split changes
-        if (sourceReaderMetrics != null && nextSplit.split() instanceof DataSplit) {
+        if (nextSplit.split() instanceof DataSplit) {
             long eventTime =
                     ((DataSplit) nextSplit.split())
                             .getLatestFileCreationEpochMillis()
                             .orElse(FileStoreSourceReaderMetrics.UNDEFINED);
-            sourceReaderMetrics.recordSnapshotUpdate(eventTime);
+            metrics.recordSnapshotUpdate(eventTime);
         }
 
         currentSplitId = nextSplit.splitId();
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkRecordsWithSplitIds.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkRecordsWithSplitIds.java
index b90313cbc..43f43abc3 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkRecordsWithSplitIds.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkRecordsWithSplitIds.java
@@ -107,12 +107,10 @@ public class FlinkRecordsWithSplitIds implements RecordsWithSplitIds<RecordItera
             RecordIterator<RowData> element,
             SourceOutput<RowData> output,
             FileStoreSourceSplitState state,
-            @Nullable FileStoreSourceReaderMetrics sourceReaderMetrics) {
+            FileStoreSourceReaderMetrics metrics) {
         long timestamp = TimestampAssigner.NO_TIMESTAMP;
-        if (sourceReaderMetrics != null
-                && sourceReaderMetrics.getLatestFileCreationTime()
-                        != FileStoreSourceReaderMetrics.UNDEFINED) {
-            timestamp = sourceReaderMetrics.getLatestFileCreationTime();
+        if (metrics.getLatestFileCreationTime() != FileStoreSourceReaderMetrics.UNDEFINED) {
+            timestamp = metrics.getLatestFileCreationTime();
         }
 
         RecordAndPosition<RowData> record;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkSource.java
index d0706a6f9..7643f7b77 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkSource.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkSource.java
@@ -56,7 +56,7 @@ public abstract class FlinkSource
         FileStoreSourceReaderMetrics sourceReaderMetrics =
                 new FileStoreSourceReaderMetrics(context.metricGroup());
         return new FileStoreSourceReader(
-                context, readBuilder.newRead(), ioManager, limit, sourceReaderMetrics);
+                context, readBuilder.newRead(), sourceReaderMetrics, ioManager, limit);
     }
 
     @Override
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedContinuousFileStoreSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedContinuousFileStoreSource.java
index 48f41b008..c00b7ea51 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedContinuousFileStoreSource.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedContinuousFileStoreSource.java
@@ -68,12 +68,12 @@ public class AlignedContinuousFileStoreSource extends ContinuousFileStoreSource
         return new AlignedSourceReader(
                 context,
                 readBuilder.newRead(),
+                sourceReaderMetrics,
                 ioManager,
                 limit,
                 new FutureCompletingBlockingQueue<>(
                         context.getConfiguration()
-                                .getInteger(SourceReaderOptions.ELEMENT_QUEUE_CAPACITY)),
-                sourceReaderMetrics);
+                                .getInteger(SourceReaderOptions.ELEMENT_QUEUE_CAPACITY)));
     }
 
     @Override
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedSourceReader.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedSourceReader.java
index e0f909344..1f0bbca31 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedSourceReader.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedSourceReader.java
@@ -53,12 +53,12 @@ public class AlignedSourceReader extends FileStoreSourceReader
     public AlignedSourceReader(
             SourceReaderContext readerContext,
             TableRead tableRead,
-            @Nullable IOManager ioManager,
+            FileStoreSourceReaderMetrics metrics,
+            IOManager ioManager,
             @Nullable Long limit,
             FutureCompletingBlockingQueue<RecordsWithSplitIds<BulkFormat.RecordIterator<RowData>>>
-                    elementsQueue,
-            @Nullable FileStoreSourceReaderMetrics sourceReaderMetrics) {
-        super(readerContext, tableRead, ioManager, limit, elementsQueue, sourceReaderMetrics);
+                    elementsQueue) {
+        super(readerContext, tableRead, metrics, ioManager, limit, elementsQueue);
         this.elementsQueue = elementsQueue;
         this.nextCheckpointId = null;
     }
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceReaderTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceReaderTest.java
index 6db17aff6..882763cf7 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceReaderTest.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceReaderTest.java
@@ -18,6 +18,8 @@
 
 package org.apache.paimon.flink.source;
 
+import org.apache.paimon.disk.IOManager;
+import org.apache.paimon.flink.source.metrics.FileStoreSourceReaderMetrics;
 import org.apache.paimon.fs.Path;
 import org.apache.paimon.fs.local.LocalFileIO;
 import org.apache.paimon.schema.Schema;
@@ -30,6 +32,12 @@ import org.apache.paimon.types.RowType;
 import org.apache.flink.api.connector.source.SourceEvent;
 import org.apache.flink.connector.testutils.source.reader.TestingReaderContext;
 import org.apache.flink.connector.testutils.source.reader.TestingReaderOutput;
+import org.apache.flink.metrics.CharacterFilter;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.Histogram;
+import org.apache.flink.metrics.Meter;
+import org.apache.flink.metrics.MetricGroup;
 import org.apache.flink.table.data.RowData;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -38,6 +46,7 @@ import org.junit.jupiter.api.io.TempDir;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 import static org.apache.paimon.flink.source.FileStoreSourceSplitSerializerTest.newSourceSplit;
 import static org.apache.paimon.mergetree.compact.MergeTreeCompactManagerTest.row;
@@ -130,12 +139,61 @@ public class FileStoreSourceReaderTest {
         return new FileStoreSourceReader(
                 context,
                 new TestChangelogDataReadWrite(tempDir.toString()).createReadWithKey(),
-                null,
-                null,
+                new FileStoreSourceReaderMetrics(new DummyMetricGroup()),
+                IOManager.create(tempDir.toString()),
                 null);
     }
 
     protected static FileStoreSourceSplit createTestFileSplit(String id) {
         return newSourceSplit(id, row(1), 0, Collections.emptyList());
     }
+
+    /** A {@link MetricGroup} for testing. */
+    public static class DummyMetricGroup implements MetricGroup {
+        public DummyMetricGroup() {}
+
+        public Counter counter(String name) {
+            return null;
+        }
+
+        public <C extends Counter> C counter(String name, C counter) {
+            return null;
+        }
+
+        public <T, G extends Gauge<T>> G gauge(String name, G gauge) {
+            return null;
+        }
+
+        public <H extends Histogram> H histogram(String name, H histogram) {
+            return null;
+        }
+
+        public <M extends Meter> M meter(String name, M meter) {
+            return null;
+        }
+
+        public MetricGroup addGroup(String name) {
+            return null;
+        }
+
+        public MetricGroup addGroup(String key, String value) {
+            return null;
+        }
+
+        public String[] getScopeComponents() {
+            return new String[0];
+        }
+
+        public Map<String, String> getAllVariables() {
+            return null;
+        }
+
+        public String getMetricIdentifier(String metricName) {
+            return null;
+        }
+
+        public String getMetricIdentifier(String metricName, CharacterFilter filter) {
+            return null;
+        }
+    }
 }
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitReaderTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitReaderTest.java
index 774a27c68..67b3c3e60 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitReaderTest.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitReaderTest.java
@@ -20,6 +20,8 @@ package org.apache.paimon.flink.source;
 
 import org.apache.paimon.KeyValue;
 import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.flink.source.FileStoreSourceReaderTest.DummyMetricGroup;
+import org.apache.paimon.flink.source.metrics.FileStoreSourceReaderMetrics;
 import org.apache.paimon.fs.Path;
 import org.apache.paimon.fs.local.LocalFileIO;
 import org.apache.paimon.io.DataFileMeta;
@@ -97,7 +99,9 @@ public class FileStoreSourceSplitReaderTest {
 
     private FileStoreSourceSplitReader createReader(TableRead tableRead, @Nullable Long limit) {
         return new FileStoreSourceSplitReader(
-                tableRead, limit == null ? null : new RecordLimiter(limit), null);
+                tableRead,
+                limit == null ? null : new RecordLimiter(limit),
+                new FileStoreSourceReaderMetrics(new DummyMetricGroup()));
     }
 
     private void innerTestOnce(int skip) throws Exception {
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedSourceReaderTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedSourceReaderTest.java
index 37e626069..c36af0f6d 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedSourceReaderTest.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedSourceReaderTest.java
@@ -18,9 +18,11 @@
 
 package org.apache.paimon.flink.source.align;
 
+import org.apache.paimon.disk.IOManager;
 import org.apache.paimon.flink.source.FileStoreSourceReader;
 import org.apache.paimon.flink.source.FileStoreSourceReaderTest;
 import org.apache.paimon.flink.source.TestChangelogDataReadWrite;
+import org.apache.paimon.flink.source.metrics.FileStoreSourceReaderMetrics;
 
 import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue;
 import org.apache.flink.connector.testutils.source.reader.TestingReaderContext;
@@ -72,9 +74,9 @@ public class AlignedSourceReaderTest extends FileStoreSourceReaderTest {
         return new AlignedSourceReader(
                 context,
                 new TestChangelogDataReadWrite(tempDir.toString()).createReadWithKey(),
+                new FileStoreSourceReaderMetrics(new DummyMetricGroup()),
+                IOManager.create(tempDir.toString()),
                 null,
-                null,
-                new FutureCompletingBlockingQueue<>(2),
-                null);
+                new FutureCompletingBlockingQueue<>(2));
     }
 }