You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/12/26 16:24:00 UTC
(incubator-paimon) branch master updated: [flink] Remove nullable in FileStoreSourceReader
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new f7bc122e3 [flink] Remove nullable in FileStoreSourceReader
f7bc122e3 is described below
commit f7bc122e35e561855c032c3a4efc317870b644e0
Author: Jingsong <ji...@gmail.com>
AuthorDate: Wed Dec 27 00:23:52 2023 +0800
[flink] Remove nullable in FileStoreSourceReader
---
.../paimon/flink/source/FileStoreSourceReader.java | 29 +++++-----
.../flink/source/FileStoreSourceSplitReader.java | 10 ++--
.../flink/source/FlinkRecordsWithSplitIds.java | 8 ++-
.../apache/paimon/flink/source/FlinkSource.java | 2 +-
.../align/AlignedContinuousFileStoreSource.java | 4 +-
.../flink/source/align/AlignedSourceReader.java | 8 +--
.../flink/source/FileStoreSourceReaderTest.java | 62 +++++++++++++++++++++-
.../source/FileStoreSourceSplitReaderTest.java | 6 ++-
.../source/align/AlignedSourceReaderTest.java | 8 +--
9 files changed, 98 insertions(+), 39 deletions(-)
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceReader.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceReader.java
index 31882406c..58e90087b 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceReader.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceReader.java
@@ -41,23 +41,23 @@ public class FileStoreSourceReader
extends SingleThreadMultiplexSourceReaderBase<
RecordIterator<RowData>, RowData, FileStoreSourceSplit, FileStoreSourceSplitState> {
- @Nullable private final IOManager ioManager;
+ private final IOManager ioManager;
+
private long lastConsumeSnapshotId = Long.MIN_VALUE;
public FileStoreSourceReader(
SourceReaderContext readerContext,
TableRead tableRead,
- @Nullable IOManager ioManager,
- @Nullable Long limit,
- @Nullable FileStoreSourceReaderMetrics sourceReaderMetrics) {
+ FileStoreSourceReaderMetrics metrics,
+ IOManager ioManager,
+ @Nullable Long limit) {
// limiter is created in SourceReader, it can be shared in all split readers
super(
() ->
new FileStoreSourceSplitReader(
- tableRead, RecordLimiter.create(limit), sourceReaderMetrics),
+ tableRead, RecordLimiter.create(limit), metrics),
(element, output, state) ->
- FlinkRecordsWithSplitIds.emitRecord(
- element, output, state, sourceReaderMetrics),
+ FlinkRecordsWithSplitIds.emitRecord(element, output, state, metrics),
readerContext.getConfiguration(),
readerContext);
this.ioManager = ioManager;
@@ -66,19 +66,18 @@ public class FileStoreSourceReader
public FileStoreSourceReader(
SourceReaderContext readerContext,
TableRead tableRead,
- @Nullable IOManager ioManager,
+ FileStoreSourceReaderMetrics metrics,
+ IOManager ioManager,
@Nullable Long limit,
FutureCompletingBlockingQueue<RecordsWithSplitIds<RecordIterator<RowData>>>
- elementsQueue,
- @Nullable FileStoreSourceReaderMetrics sourceReaderMetrics) {
+ elementsQueue) {
super(
elementsQueue,
() ->
new FileStoreSourceSplitReader(
- tableRead, RecordLimiter.create(limit), sourceReaderMetrics),
+ tableRead, RecordLimiter.create(limit), metrics),
(element, output, state) ->
- FlinkRecordsWithSplitIds.emitRecord(
- element, output, state, sourceReaderMetrics),
+ FlinkRecordsWithSplitIds.emitRecord(element, output, state, metrics),
readerContext.getConfiguration(),
readerContext);
this.ioManager = ioManager;
@@ -130,8 +129,6 @@ public class FileStoreSourceReader
@Override
public void close() throws Exception {
super.close();
- if (ioManager != null) {
- ioManager.close();
- }
+ ioManager.close();
}
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceSplitReader.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceSplitReader.java
index 7260650be..c70c339d7 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceSplitReader.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceSplitReader.java
@@ -65,19 +65,19 @@ public class FileStoreSourceSplitReader
private RecordIterator<InternalRow> currentFirstBatch;
private boolean paused;
- private final FileStoreSourceReaderMetrics sourceReaderMetrics;
+ private final FileStoreSourceReaderMetrics metrics;
public FileStoreSourceSplitReader(
TableRead tableRead,
@Nullable RecordLimiter limiter,
- @Nullable FileStoreSourceReaderMetrics sourceReaderMetrics) {
+ FileStoreSourceReaderMetrics metrics) {
this.tableRead = tableRead;
this.limiter = limiter;
this.splits = new LinkedList<>();
this.pool = new Pool<>(1);
this.pool.add(new FileStoreRecordIterator());
this.paused = false;
- this.sourceReaderMetrics = sourceReaderMetrics;
+ this.metrics = metrics;
}
@Override
@@ -178,12 +178,12 @@ public class FileStoreSourceSplitReader
}
// update metric when split changes
- if (sourceReaderMetrics != null && nextSplit.split() instanceof DataSplit) {
+ if (nextSplit.split() instanceof DataSplit) {
long eventTime =
((DataSplit) nextSplit.split())
.getLatestFileCreationEpochMillis()
.orElse(FileStoreSourceReaderMetrics.UNDEFINED);
- sourceReaderMetrics.recordSnapshotUpdate(eventTime);
+ metrics.recordSnapshotUpdate(eventTime);
}
currentSplitId = nextSplit.splitId();
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkRecordsWithSplitIds.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkRecordsWithSplitIds.java
index b90313cbc..43f43abc3 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkRecordsWithSplitIds.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkRecordsWithSplitIds.java
@@ -107,12 +107,10 @@ public class FlinkRecordsWithSplitIds implements RecordsWithSplitIds<RecordItera
RecordIterator<RowData> element,
SourceOutput<RowData> output,
FileStoreSourceSplitState state,
- @Nullable FileStoreSourceReaderMetrics sourceReaderMetrics) {
+ FileStoreSourceReaderMetrics metrics) {
long timestamp = TimestampAssigner.NO_TIMESTAMP;
- if (sourceReaderMetrics != null
- && sourceReaderMetrics.getLatestFileCreationTime()
- != FileStoreSourceReaderMetrics.UNDEFINED) {
- timestamp = sourceReaderMetrics.getLatestFileCreationTime();
+ if (metrics.getLatestFileCreationTime() != FileStoreSourceReaderMetrics.UNDEFINED) {
+ timestamp = metrics.getLatestFileCreationTime();
}
RecordAndPosition<RowData> record;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkSource.java
index d0706a6f9..7643f7b77 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkSource.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkSource.java
@@ -56,7 +56,7 @@ public abstract class FlinkSource
FileStoreSourceReaderMetrics sourceReaderMetrics =
new FileStoreSourceReaderMetrics(context.metricGroup());
return new FileStoreSourceReader(
- context, readBuilder.newRead(), ioManager, limit, sourceReaderMetrics);
+ context, readBuilder.newRead(), sourceReaderMetrics, ioManager, limit);
}
@Override
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedContinuousFileStoreSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedContinuousFileStoreSource.java
index 48f41b008..c00b7ea51 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedContinuousFileStoreSource.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedContinuousFileStoreSource.java
@@ -68,12 +68,12 @@ public class AlignedContinuousFileStoreSource extends ContinuousFileStoreSource
return new AlignedSourceReader(
context,
readBuilder.newRead(),
+ sourceReaderMetrics,
ioManager,
limit,
new FutureCompletingBlockingQueue<>(
context.getConfiguration()
- .getInteger(SourceReaderOptions.ELEMENT_QUEUE_CAPACITY)),
- sourceReaderMetrics);
+ .getInteger(SourceReaderOptions.ELEMENT_QUEUE_CAPACITY)));
}
@Override
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedSourceReader.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedSourceReader.java
index e0f909344..1f0bbca31 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedSourceReader.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedSourceReader.java
@@ -53,12 +53,12 @@ public class AlignedSourceReader extends FileStoreSourceReader
public AlignedSourceReader(
SourceReaderContext readerContext,
TableRead tableRead,
- @Nullable IOManager ioManager,
+ FileStoreSourceReaderMetrics metrics,
+ IOManager ioManager,
@Nullable Long limit,
FutureCompletingBlockingQueue<RecordsWithSplitIds<BulkFormat.RecordIterator<RowData>>>
- elementsQueue,
- @Nullable FileStoreSourceReaderMetrics sourceReaderMetrics) {
- super(readerContext, tableRead, ioManager, limit, elementsQueue, sourceReaderMetrics);
+ elementsQueue) {
+ super(readerContext, tableRead, metrics, ioManager, limit, elementsQueue);
this.elementsQueue = elementsQueue;
this.nextCheckpointId = null;
}
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceReaderTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceReaderTest.java
index 6db17aff6..882763cf7 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceReaderTest.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceReaderTest.java
@@ -18,6 +18,8 @@
package org.apache.paimon.flink.source;
+import org.apache.paimon.disk.IOManager;
+import org.apache.paimon.flink.source.metrics.FileStoreSourceReaderMetrics;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.schema.Schema;
@@ -30,6 +32,12 @@ import org.apache.paimon.types.RowType;
import org.apache.flink.api.connector.source.SourceEvent;
import org.apache.flink.connector.testutils.source.reader.TestingReaderContext;
import org.apache.flink.connector.testutils.source.reader.TestingReaderOutput;
+import org.apache.flink.metrics.CharacterFilter;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.Histogram;
+import org.apache.flink.metrics.Meter;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.table.data.RowData;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -38,6 +46,7 @@ import org.junit.jupiter.api.io.TempDir;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
import static org.apache.paimon.flink.source.FileStoreSourceSplitSerializerTest.newSourceSplit;
import static org.apache.paimon.mergetree.compact.MergeTreeCompactManagerTest.row;
@@ -130,12 +139,61 @@ public class FileStoreSourceReaderTest {
return new FileStoreSourceReader(
context,
new TestChangelogDataReadWrite(tempDir.toString()).createReadWithKey(),
- null,
- null,
+ new FileStoreSourceReaderMetrics(new DummyMetricGroup()),
+ IOManager.create(tempDir.toString()),
null);
}
protected static FileStoreSourceSplit createTestFileSplit(String id) {
return newSourceSplit(id, row(1), 0, Collections.emptyList());
}
+
+ /** A {@link MetricGroup} for testing. */
+ public static class DummyMetricGroup implements MetricGroup {
+ public DummyMetricGroup() {}
+
+ public Counter counter(String name) {
+ return null;
+ }
+
+ public <C extends Counter> C counter(String name, C counter) {
+ return null;
+ }
+
+ public <T, G extends Gauge<T>> G gauge(String name, G gauge) {
+ return null;
+ }
+
+ public <H extends Histogram> H histogram(String name, H histogram) {
+ return null;
+ }
+
+ public <M extends Meter> M meter(String name, M meter) {
+ return null;
+ }
+
+ public MetricGroup addGroup(String name) {
+ return null;
+ }
+
+ public MetricGroup addGroup(String key, String value) {
+ return null;
+ }
+
+ public String[] getScopeComponents() {
+ return new String[0];
+ }
+
+ public Map<String, String> getAllVariables() {
+ return null;
+ }
+
+ public String getMetricIdentifier(String metricName) {
+ return null;
+ }
+
+ public String getMetricIdentifier(String metricName, CharacterFilter filter) {
+ return null;
+ }
+ }
}
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitReaderTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitReaderTest.java
index 774a27c68..67b3c3e60 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitReaderTest.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitReaderTest.java
@@ -20,6 +20,8 @@ package org.apache.paimon.flink.source;
import org.apache.paimon.KeyValue;
import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.flink.source.FileStoreSourceReaderTest.DummyMetricGroup;
+import org.apache.paimon.flink.source.metrics.FileStoreSourceReaderMetrics;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.io.DataFileMeta;
@@ -97,7 +99,9 @@ public class FileStoreSourceSplitReaderTest {
private FileStoreSourceSplitReader createReader(TableRead tableRead, @Nullable Long limit) {
return new FileStoreSourceSplitReader(
- tableRead, limit == null ? null : new RecordLimiter(limit), null);
+ tableRead,
+ limit == null ? null : new RecordLimiter(limit),
+ new FileStoreSourceReaderMetrics(new DummyMetricGroup()));
}
private void innerTestOnce(int skip) throws Exception {
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedSourceReaderTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedSourceReaderTest.java
index 37e626069..c36af0f6d 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedSourceReaderTest.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedSourceReaderTest.java
@@ -18,9 +18,11 @@
package org.apache.paimon.flink.source.align;
+import org.apache.paimon.disk.IOManager;
import org.apache.paimon.flink.source.FileStoreSourceReader;
import org.apache.paimon.flink.source.FileStoreSourceReaderTest;
import org.apache.paimon.flink.source.TestChangelogDataReadWrite;
+import org.apache.paimon.flink.source.metrics.FileStoreSourceReaderMetrics;
import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue;
import org.apache.flink.connector.testutils.source.reader.TestingReaderContext;
@@ -72,9 +74,9 @@ public class AlignedSourceReaderTest extends FileStoreSourceReaderTest {
return new AlignedSourceReader(
context,
new TestChangelogDataReadWrite(tempDir.toString()).createReadWithKey(),
+ new FileStoreSourceReaderMetrics(new DummyMetricGroup()),
+ IOManager.create(tempDir.toString()),
null,
- null,
- new FutureCompletingBlockingQueue<>(2),
- null);
+ new FutureCompletingBlockingQueue<>(2));
}
}