You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by di...@apache.org on 2026/05/19 06:13:23 UTC
(doris) branch master updated: [improve](streaming-job) async chunk splitting for cdc source job (#63079)
This is an automated email from the ASF dual-hosted git repository.
JNSimba pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new be49e67d7b0 [improve](streaming-job) async chunk splitting for cdc source job (#63079)
be49e67d7b0 is described below
commit be49e67d7b003c2193cab1746d23acc402d2f86b
Author: wudi <wu...@selectdb.com>
AuthorDate: Tue May 19 14:13:15 2026 +0800
[improve](streaming-job) async chunk splitting for cdc source job (#63079)
## Summary
`StreamingInsertJob` (CDC FROM-TO and `cdc_stream` TVF paths) used to
call `splitChunks()` synchronously inside `CREATE STREAMING JOB`, asking
cdc_client to cut every chunk of every table before returning. On
large/non-uniform PK tables this can take 30+ minutes — far beyond the
BE→cdc_client BRPC 60s timeout, and the SQL client blocks the whole
time.
This PR makes splitting tick-driven by the FE scheduler:
- `CREATE` returns immediately; no more synchronous `splitChunks()`.
- Each scheduler tick `advanceSplits()` issues one short fetchSplits RPC
(default `batchSize=100`) and pushes that batch into `remainingSplits`.
Tasks dispatch as soon as the first batch lands, so end-to-end
first-byte latency stays close to flink-cdc's.
- cdc_client is **stateless** — every RPC reconstructs `ChunkSplitter`
from the `(currentSplittingTable, nextSplitStart, nextSplitId)` triple
supplied by FE; flink-cdc internals are untouched (uses the public
`ChunkSplitter` API only).
- Crash recovery uses three sources of truth:
- editlog persists `committedSplitProgress` (3-field `SplitProgress`) +
existing `chunkHighWatermarkMap` / `binlogOffsetPersist`
- `streaming_job_meta` system table holds full `chunk_list` JSON per
table (UPSERT each `advanceSplits`)
- cdc_client memory holds nothing
- Both FROM-TO (multi-table) and TVF (single-table) paths share the same
`SourceOffsetProvider#initSplitProgress` / `noMoreSplits` /
`advanceSplits` interface;
`StreamingJobSchedulerTask.handlePendingState` pre-advances one batch so
the first task doesn't wait a full `max_interval`.
---
.../postgresql/postgresql-14.yaml.tpl | 4 +-
.../main/java/org/apache/doris/common/Config.java | 3 +
.../job/cdc/request/FetchTableSplitsRequest.java | 12 +
.../insert/streaming/StreamingInsertJob.java | 39 +-
.../streaming/StreamingJobSchedulerTask.java | 7 +
.../doris/job/offset/SourceOffsetProvider.java | 24 +-
.../job/offset/jdbc/JdbcSourceOffsetProvider.java | 503 ++++++++++++++++-----
.../offset/jdbc/JdbcTvfSourceOffsetProvider.java | 60 ++-
.../apache/doris/job/util/StreamingJobUtils.java | 72 ++-
.../JdbcSourceOffsetProviderAsyncSplitTest.java | 486 ++++++++++++++++++++
.../doris/job/offset/jdbc/SplitProgressTest.java | 60 +++
.../source/reader/JdbcIncrementalSourceReader.java | 216 ++++-----
.../source/reader/mysql/MySqlSourceReader.java | 212 +++++----
.../test_streaming_mysql_job_async_split.groovy | 160 +++++++
...t_streaming_mysql_job_async_split_uneven.groovy | 195 ++++++++
...ing_postgres_job_async_split_multi_table.groovy | 178 ++++++++
...ng_postgres_job_async_split_pause_resume.groovy | 172 +++++++
...treaming_postgres_job_async_split_uneven.groovy | 198 ++++++++
...stgres_job_async_split_uneven_restart_fe.groovy | 207 +++++++++
...streaming_postgres_job_binlog_restart_fe.groovy | 191 ++++++++
...st_streaming_postgres_job_special_offset.groovy | 15 +-
...ming_job_cdc_stream_postgres_async_split.groovy | 193 ++++++++
...c_stream_postgres_async_split_restart_fe.groovy | 218 +++++++++
23 files changed, 3038 insertions(+), 387 deletions(-)
diff --git a/docker/thirdparties/docker-compose/postgresql/postgresql-14.yaml.tpl b/docker/thirdparties/docker-compose/postgresql/postgresql-14.yaml.tpl
index 1012a65b1f2..54758285290 100644
--- a/docker/thirdparties/docker-compose/postgresql/postgresql-14.yaml.tpl
+++ b/docker/thirdparties/docker-compose/postgresql/postgresql-14.yaml.tpl
@@ -40,9 +40,9 @@ services:
- "-c"
- "wal_level=logical"
- "-c"
- - "max_wal_senders=30"
+ - "max_wal_senders=64"
- "-c"
- - "max_replication_slots=30"
+ - "max_replication_slots=64"
- "-c"
- "ssl=on"
- "-c"
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 2dec4a2222a..5c242bd25fa 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1182,6 +1182,9 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static int streaming_cdc_heavy_rpc_timeout_sec = 600;
+ @ConfField(mutable = true, masterOnly = true)
+ public static int streaming_cdc_fetch_splits_batch_size = 100;
+
/**
* the max timeout of get kafka meta.
*/
diff --git a/fe/fe-common/src/main/java/org/apache/doris/job/cdc/request/FetchTableSplitsRequest.java b/fe/fe-common/src/main/java/org/apache/doris/job/cdc/request/FetchTableSplitsRequest.java
index a54b2630734..83b0ec6c248 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/job/cdc/request/FetchTableSplitsRequest.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/job/cdc/request/FetchTableSplitsRequest.java
@@ -33,6 +33,18 @@ import java.util.Map;
public class FetchTableSplitsRequest extends JobBaseConfig {
private String snapshotTable;
+ /**
+ * The next split start, null = fresh start (START_BOUND).
+ * cdc_client takes to construct ChunkSplitterState.
+ */
+ private Object[] nextSplitStart;
+
+ /** Next split id; null = 0 (fresh start). */
+ private Integer nextSplitId;
+
+ /** Splits to fetch in this RPC; null lets cdc_client use its default. */
+ private Integer batchSize;
+
public FetchTableSplitsRequest(Long jobId, String name,
Map<String, String> sourceProperties, String frontendAddress, String snapshotTable) {
super(jobId.toString(), name, sourceProperties, frontendAddress);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java
index 7e6c2c3830b..cf431fd7601 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java
@@ -94,6 +94,7 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
@@ -174,6 +175,11 @@ public class StreamingInsertJob extends AbstractJob<StreamingJobSchedulerTask, M
@SerializedName("ccn")
private volatile String cloudCluster;
+ /** Source tables this job syncs. */
+ @Getter
+ @SerializedName("st")
+ private List<String> syncTables;
+
// The sampling window starts at the beginning of the sampling window.
// If the error rate exceeds `max_filter_ratio` within the window, the sampling fails.
@Setter
@@ -238,6 +244,7 @@ public class StreamingInsertJob extends AbstractJob<StreamingJobSchedulerTask, M
init();
checkRequiredSourceProperties();
List<String> createTbls = createTableIfNotExists();
+ this.syncTables = createTbls;
if (sourceProperties.get(DataSourceConfigKeys.INCLUDE_TABLES) == null) {
// cdc need the final includeTables
String includeTables = String.join(",", createTbls);
@@ -246,8 +253,7 @@ public class StreamingInsertJob extends AbstractJob<StreamingJobSchedulerTask, M
StreamingJobUtils.resolveAndValidateSource(
dataSourceType, sourceProperties, String.valueOf(getJobId()), createTbls);
this.offsetProvider = createOffsetProvider(getConvertedSourceProperties());
- JdbcSourceOffsetProvider rdsOffsetProvider = (JdbcSourceOffsetProvider) this.offsetProvider;
- rdsOffsetProvider.splitChunks(createTbls);
+ this.offsetProvider.initOnCreate(this.syncTables);
} catch (Exception ex) {
log.warn("init streaming job for {} failed", dataSourceType, ex);
throw new RuntimeException(ex.getMessage());
@@ -396,10 +402,13 @@ public class StreamingInsertJob extends AbstractJob<StreamingJobSchedulerTask, M
this.originTvfProps = currentTvf.getProperties().getMap();
this.offsetProvider = createOffsetProvider(sourceProperties);
this.offsetProvider.ensureInitialized(getJobId(), originTvfProps);
+ // cdc_stream TVF has TABLE; S3 TVF doesn't, so syncTables stays null there.
+ String tvfTable = originTvfProps.get(DataSourceConfigKeys.TABLE);
+ this.syncTables = tvfTable == null ? null : Collections.singletonList(tvfTable);
// Validate source-side resources (e.g. PG slot/publication ownership) once at job
// creation so conflicts fail fast. No-op for standalone cdc_stream TVF (no job).
StreamingJobUtils.validateTvfSource(tvfType, originTvfProps, String.valueOf(getJobId()));
- this.offsetProvider.initOnCreate();
+ this.offsetProvider.initOnCreate(this.syncTables);
// validate offset props, only for s3 cause s3 tvf no offset prop
if (jobProperties.getOffsetProperty() != null
&& S3TableValuedFunction.NAME.equalsIgnoreCase(tvfType)) {
@@ -718,6 +727,28 @@ public class StreamingInsertJob extends AbstractJob<StreamingJobSchedulerTask, M
}
}
+ /**
+ * Advance one batch of split fetching if there are still tables to split.
+ * Called by scheduler each tick (PENDING/RUNNING). Mirrors fetchMeta error handling.
+ */
+ public void advanceSplitsIfNeed() throws JobException {
+ if (offsetProvider.noMoreSplits()) {
+ return;
+ }
+ try {
+ offsetProvider.advanceSplits();
+ } catch (Exception ex) {
+ log.warn("advance splits failed, job id: {}", getJobId(), ex);
+ if (this.getFailureReason() == null
+ || !InternalErrorCode.MANUAL_PAUSE_ERR.equals(this.getFailureReason().getCode())) {
+ this.setFailureReason(new FailureReason(
+ InternalErrorCode.GET_REMOTE_DATA_ERROR,
+ "Failed to advance splits, " + ex.getMessage()));
+ this.updateJobStatus(JobStatus.PAUSED);
+ }
+ }
+ }
+
public boolean needScheduleTask() {
readLock();
try {
@@ -1487,6 +1518,8 @@ public class StreamingInsertJob extends AbstractJob<StreamingJobSchedulerTask, M
public void replayOffsetProviderIfNeed() throws JobException {
if (offsetProvider != null) {
+ // when fe restart, offsetProvider.jobId/sourceProperties may be null
+ offsetProvider.ensureInitialized(getJobId(), getProviderProps());
offsetProvider.replayIfNeed(this);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobSchedulerTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobSchedulerTask.java
index b1948dd4bd5..0f6bcba892b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobSchedulerTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobSchedulerTask.java
@@ -66,6 +66,12 @@ public class StreamingJobSchedulerTask extends AbstractTask {
}
}
streamingInsertJob.replayOffsetProviderIfNeed();
+ // Pre-advance one batch so the first task has splits to consume
+ streamingInsertJob.advanceSplitsIfNeed();
+ if (streamingInsertJob.getJobStatus() == JobStatus.PAUSED) {
+ // advanceSplits failed and paused the job; skip task dispatch this tick.
+ return;
+ }
if (streamingInsertJob.hasReachedEnd()) {
// Source already fully consumed (e.g. snapshot-only mode recovered after FE restart).
// Transition directly to FINISHED without creating a new task.
@@ -80,6 +86,7 @@ public class StreamingJobSchedulerTask extends AbstractTask {
private void handleRunningState() throws JobException {
streamingInsertJob.processTimeoutTasks();
streamingInsertJob.fetchMeta();
+ streamingInsertJob.advanceSplitsIfNeed();
}
private void autoResumeHandler() throws JobException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/offset/SourceOffsetProvider.java b/fe/fe-core/src/main/java/org/apache/doris/job/offset/SourceOffsetProvider.java
index 87eca253c46..a6f9e582bff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/offset/SourceOffsetProvider.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/offset/SourceOffsetProvider.java
@@ -45,11 +45,11 @@ public interface SourceOffsetProvider {
default void ensureInitialized(Long jobId, Map<String, String> originTvfProps) throws JobException {}
/**
- * Performs one-time initialization that must run only on fresh job creation, not on FE restart.
- * For example, fetching and persisting snapshot splits to the meta table.
+ * One-time initialization on fresh job creation (not on FE restart). Subclasses may
+ * initialize split progress, fetch initial splits, or open remote readers.
* Default: no-op (most providers need no extra setup).
*/
- default void initOnCreate() throws JobException {}
+ default void initOnCreate(List<String> syncTables) throws JobException {}
/**
* Get next offset to consume
@@ -180,6 +180,24 @@ public interface SourceOffsetProvider {
return false;
}
+ /**
+ * Advance one batch of split fetching, called by scheduler each tick during PENDING/RUNNING.
+ * For providers without async splitting work (e.g. S3, Kafka), default is no-op.
+ * Aligned with flink-cdc SnapshotSplitAssigner naming.
+ *
+ * @throws JobException if fetching splits fails fatally
+ */
+ default void advanceSplits() throws JobException {}
+
+ /**
+ * Returns true if no more splits will be produced.
+ * For providers without splitting concept, always returns true.
+ * Aligned with flink-cdc SnapshotSplitAssigner.noMoreSplits() naming.
+ */
+ default boolean noMoreSplits() {
+ return true;
+ }
+
/**
* Get the lag of the data source in seconds.
* For CDC sources, lag = (now - last consumed event timestamp) in seconds.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java
index bada3a1d7a7..9cd5ad3fefa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java
@@ -58,11 +58,15 @@ import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
-import java.util.LinkedHashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
@@ -95,10 +99,23 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
@SerializedName("ts")
String tableSchemas;
+ /** Split progress (task-commit view). */
+ @SerializedName("csp")
+ SplitProgress committedSplitProgress;
+
volatile boolean hasMoreData = true;
transient volatile String cloudCluster;
+ /** Split progress (cdc-fetch view), >= committedSplitProgress. Rebuilt on restart. */
+ transient SplitProgress cdcSplitProgress = new SplitProgress();
+
+ /** Cache of Job.syncTables, set by initSplitProgress / replayIfNeed. */
+ transient List<String> cachedSyncTables;
+
+ /** Guards cdcSplitProgress/committedSplitProgress/remainingSplits/finishedSplits. */
+ protected final transient Object splitsLock = new Object();
+
/**
* No-arg constructor for subclass use.
*/
@@ -135,22 +152,26 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
@Override
public Offset getNextOffset(StreamingJobProperties jobProps, Map<String, String> properties) {
- JdbcOffset nextOffset = new JdbcOffset();
- if (!remainingSplits.isEmpty()) {
- int splitsNum = Math.min(remainingSplits.size(), snapshotParallelism);
- List<SnapshotSplit> snapshotSplits = new ArrayList<>(remainingSplits.subList(0, splitsNum));
- nextOffset.setSplits(snapshotSplits);
- return nextOffset;
- } else if (currentOffset != null && currentOffset.snapshotSplit()) {
- // initial mode: snapshot to binlog
- // snapshot-only mode must be intercepted by hasReachedEnd() before reaching here
- BinlogSplit binlogSplit = new BinlogSplit();
- binlogSplit.setFinishedSplits(finishedSplits);
- nextOffset.setSplits(Collections.singletonList(binlogSplit));
- return nextOffset;
- } else {
- // only binlog
- return currentOffset == null ? new JdbcOffset(Collections.singletonList(new BinlogSplit())) : currentOffset;
+ synchronized (splitsLock) {
+ JdbcOffset nextOffset = new JdbcOffset();
+ if (!remainingSplits.isEmpty()) {
+ int splitsNum = Math.min(remainingSplits.size(), snapshotParallelism);
+ List<SnapshotSplit> snapshotSplits = new ArrayList<>(remainingSplits.subList(0, splitsNum));
+ nextOffset.setSplits(snapshotSplits);
+ return nextOffset;
+ } else if (currentOffset != null && currentOffset.snapshotSplit() && noMoreSplits()) {
+ // initial mode: snapshot to binlog. noMoreSplits() guards against switching while
+ // splitting is still in progress (remainingSplits empty doesn't mean fully cut).
+ // snapshot-only mode is intercepted by hasReachedEnd() before reaching here.
+ BinlogSplit binlogSplit = new BinlogSplit();
+ binlogSplit.setFinishedSplits(new ArrayList<>(finishedSplits));
+ nextOffset.setSplits(Collections.singletonList(binlogSplit));
+ return nextOffset;
+ } else {
+ // only binlog
+ return currentOffset == null
+ ? new JdbcOffset(Collections.singletonList(new BinlogSplit())) : currentOffset;
+ }
}
}
@@ -194,26 +215,35 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
public void updateOffset(Offset offset) {
this.currentOffset = (JdbcOffset) offset;
if (currentOffset.snapshotSplit()) {
- List<? extends AbstractSourceSplit> splits = currentOffset.getSplits();
- for (AbstractSourceSplit split : splits) {
- SnapshotSplit snapshotSplit = (SnapshotSplit) split;
- String splitId = split.getSplitId();
- boolean remove = remainingSplits.removeIf(v -> {
- if (v.getSplitId().equals(splitId)) {
- snapshotSplit.setTableId(v.getTableId());
- snapshotSplit.setSplitKey(v.getSplitKey());
- snapshotSplit.setSplitStart(v.getSplitStart());
- snapshotSplit.setSplitEnd(v.getSplitEnd());
- return true;
+ synchronized (splitsLock) {
+ List<? extends AbstractSourceSplit> splits = currentOffset.getSplits();
+ for (AbstractSourceSplit split : splits) {
+ SnapshotSplit snapshotSplit = (SnapshotSplit) split;
+ String splitId = split.getSplitId();
+ boolean remove = remainingSplits.removeIf(v -> {
+ if (v.getSplitId().equals(splitId)) {
+ snapshotSplit.setTableId(v.getTableId());
+ snapshotSplit.setSplitKey(v.getSplitKey());
+ snapshotSplit.setSplitStart(v.getSplitStart());
+ snapshotSplit.setSplitEnd(v.getSplitEnd());
+ return true;
+ }
+ return false;
+ });
+ if (remove) {
+ finishedSplits.add(snapshotSplit);
+ chunkHighWatermarkMap.computeIfAbsent(snapshotSplit.getTableId(), k -> new HashMap<>())
+ .put(snapshotSplit.getSplitId(), snapshotSplit.getHighWatermark());
+
+ // Advance committedSplitProgress to this committed chunk.
+ if (committedSplitProgress != null) {
+ applySplitToProgress(committedSplitProgress, snapshotSplit,
+ getTableName(snapshotSplit.getTableId()));
+ }
+ } else {
+ // Replay before remainingSplits is restored, or a duplicate commit.
+ log.warn("Cannot find snapshot split {} in remainingSplits for job {}", splitId, getJobId());
}
- return false;
- });
- if (remove) {
- finishedSplits.add(snapshotSplit);
- chunkHighWatermarkMap.computeIfAbsent(snapshotSplit.getTableId(), k -> new HashMap<>())
- .put(snapshotSplit.getSplitId(), snapshotSplit.getHighWatermark());
- } else {
- log.warn("Cannot find snapshot split {} in remainingSplits for job {}", splitId, getJobId());
}
}
} else {
@@ -278,19 +308,26 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
return true;
}
- if (currentOffset.snapshotSplit()) {
- if (isSnapshotOnlyMode() && remainingSplits.isEmpty()) {
- return false;
+ synchronized (splitsLock) {
+ if (currentOffset.snapshotSplit()) {
+ if (!remainingSplits.isEmpty()) {
+ return true;
+ }
+ // Splitting still in progress: wait for next tick.
+ if (!noMoreSplits()) {
+ return false;
+ }
+ // Splitting done: snapshot-only completes; initial mode falls through to binlog.
+ return !isSnapshotOnlyMode();
}
- return true;
- }
- if (!hasMoreData) {
- return false;
- }
+ if (!hasMoreData) {
+ return false;
+ }
- if (CollectionUtils.isNotEmpty(remainingSplits)) {
- return true;
+ if (CollectionUtils.isNotEmpty(remainingSplits)) {
+ return true;
+ }
}
if (MapUtils.isEmpty(endBinlogOffset)) {
return false;
@@ -420,6 +457,10 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
*/
@Override
public void replayIfNeed(StreamingInsertJob job) throws JobException {
+ synchronized (splitsLock) {
+ this.cachedSyncTables = job.getSyncTables();
+ }
+
String offsetProviderPersist = job.getOffsetProviderPersist();
if (offsetProviderPersist != null) {
JdbcSourceOffsetProvider replayFromPersist = GsonUtils.GSON.fromJson(offsetProviderPersist,
@@ -427,6 +468,11 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
this.binlogOffsetPersist = replayFromPersist.getBinlogOffsetPersist();
this.chunkHighWatermarkMap = replayFromPersist.getChunkHighWatermarkMap();
this.tableSchemas = replayFromPersist.getTableSchemas();
+ synchronized (splitsLock) {
+ this.committedSplitProgress = replayFromPersist.getCommittedSplitProgress() != null
+ ? replayFromPersist.getCommittedSplitProgress() : new SplitProgress();
+ this.cdcSplitProgress = this.committedSplitProgress.copy();
+ }
log.info("Replaying offset provider for job {}, binlogOffset size {}, chunkHighWatermark size {}",
getJobId(),
binlogOffsetPersist == null ? 0 : binlogOffsetPersist.size(),
@@ -439,23 +485,23 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
if (MapUtils.isNotEmpty(chunkHighWatermarkMap) && MapUtils.isNotEmpty(snapshotSplits)) {
List<SnapshotSplit> lastSnapshotSplits =
recalculateRemainingSplits(chunkHighWatermarkMap, snapshotSplits);
+ resumeCdcSplitProgressFromSplits();
if (this.remainingSplits.isEmpty()) {
if (!lastSnapshotSplits.isEmpty()) {
currentOffset = new JdbcOffset();
currentOffset.setSplits(lastSnapshotSplits);
- } else if (!isSnapshotOnlyMode()) {
+ } else if (!isSnapshotOnlyMode() && noMoreSplits()) {
// initial mode: rebuild binlog split for snapshot-to-binlog transition
currentOffset = new JdbcOffset();
BinlogSplit binlogSplit = new BinlogSplit();
binlogSplit.setFinishedSplits(finishedSplits);
currentOffset.setSplits(Collections.singletonList(binlogSplit));
- } else {
- // snapshot-only completed: leave currentOffset as null,
- // hasReachedEnd() detects completion via finishedSplits
+ } else if (isSnapshotOnlyMode()) {
log.info("Replaying offset provider for job {}: snapshot-only mode completed,"
+ " finishedSplits={}, skip currentOffset restoration",
getJobId(), finishedSplits.size());
}
+ // else: splitter mid-flight, keep currentOffset as snapshotSplit.
}
}
}
@@ -470,11 +516,63 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
log.info("Replaying offset provider for job {}, offsetProviderPersist is empty", getJobId());
Map<String, List<SnapshotSplit>> snapshotSplits = StreamingJobUtils.restoreSplitsToJob(job.getJobId());
recalculateRemainingSplits(new HashMap<>(), snapshotSplits);
+ resumeCdcSplitProgressFromSplits();
} else {
log.info("No need to replay offset provider for job {}", getJobId());
}
}
+ /** Rebuild cdcSplitProgress after restart so advanceSplits resumes mid-table instead of skipping it. */
+ protected void resumeCdcSplitProgressFromSplits() {
+ synchronized (splitsLock) {
+ if (cachedSyncTables == null || cachedSyncTables.isEmpty()) {
+ return;
+ }
+ SnapshotSplit mid = findResumeMidSplit(cachedSyncTables, finishedSplits, remainingSplits);
+ if (mid != null) {
+ applySplitToProgress(cdcSplitProgress, mid, getTableName(mid.getTableId()));
+ } else {
+ clearProgress(cdcSplitProgress);
+ }
+ log.info("Resume cdcSplitProgress for job {}: finishedSplits={}, remainingSplits={}, "
+ + "cdcSplitProgress=(table={}, nextStart={}, nextSplitId={})",
+ getJobId(), finishedSplits.size(), remainingSplits.size(),
+ cdcSplitProgress.getCurrentSplittingTable(),
+ Arrays.toString(cdcSplitProgress.getNextSplitStart()),
+ cdcSplitProgress.getNextSplitId());
+ }
+ }
+
+ /**
+ * Find the at-most-one table cut to mid (its largest-id split has non-null splitEnd).
+ * Returns null when every table in {@code syncTables} is either untouched or fully cut.
+ */
+ static SnapshotSplit findResumeMidSplit(List<String> syncTables,
+ List<SnapshotSplit> finishedSplits,
+ List<SnapshotSplit> remainingSplits) {
+ // Map keys are bare names so syncTables (bare) and SnapshotSplit.tableId (qualified) align.
+ Map<String, SnapshotSplit> lastPerTable = new HashMap<>();
+ pickLastById(finishedSplits, lastPerTable);
+ pickLastById(remainingSplits, lastPerTable);
+ for (String tbl : syncTables) {
+ SnapshotSplit last = lastPerTable.get(getTableName(tbl));
+ if (last != null && last.getSplitEnd() != null && last.getSplitEnd().length > 0) {
+ return last;
+ }
+ }
+ return null;
+ }
+
+ private static void pickLastById(List<SnapshotSplit> splits, Map<String, SnapshotSplit> out) {
+ for (SnapshotSplit s : splits) {
+ String key = getTableName(s.getTableId());
+ SnapshotSplit prev = out.get(key);
+ if (prev == null || splitIdOf(s.getSplitId()) > splitIdOf(prev.getSplitId())) {
+ out.put(key, s);
+ }
+ }
+ }
+
/**
* Assign the HW value to the synchronized Split,
* and remove the Split from remainSplit and place it in finishedSplit.
@@ -520,7 +618,7 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
return lastSnapshotSplits;
}
- private String getTableName(String tableId) {
+ private static String getTableName(String tableId) {
if (tableId == null) {
return null;
}
@@ -533,78 +631,239 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
return GsonUtils.GSON.toJson(this);
}
- public void splitChunks(List<String> createTbls) throws JobException {
- // todo: When splitting takes a long time, it needs to be changed to asynchronous.
- if (checkNeedSplitChunks(sourceProperties)) {
- Map<String, List<SnapshotSplit>> tableSplits = new LinkedHashMap<>();
- for (String tbl : createTbls) {
- List<SnapshotSplit> snapshotSplits = requestTableSplits(tbl);
- tableSplits.put(tbl, snapshotSplits);
- }
- // save chunk list to system table
- saveChunkMeta(tableSplits);
- this.remainingSplits = tableSplits.values().stream()
- .flatMap(List::stream)
- .collect(Collectors.toList());
- } else {
- // The source reader is automatically initialized when the split is obtained.
- // In latest mode, a separate init is required.init source reader
+ // ============ Async split progress (driven by scheduler each tick) ============
+
+ /**
+ * One-time setup at CREATE.
+ * - initial/snapshot mode: init split progress; scheduler will drive advanceSplits() each tick.
+ * - latest mode (and other non-splitting modes): open the remote reader (e.g. PG slot) so the
+ * binlog phase can start immediately; no snapshot splitting will happen.
+ */
+ @Override
+ public void initOnCreate(List<String> syncTables) throws JobException {
+ if (!checkNeedSplitChunks(sourceProperties)) {
initSourceReader();
+ return;
+ }
+ synchronized (splitsLock) {
+ this.cachedSyncTables = syncTables;
+ this.committedSplitProgress = new SplitProgress();
+ this.cdcSplitProgress = new SplitProgress();
+ }
+ }
+
+ @Override
+ public boolean noMoreSplits() {
+ if (!checkNeedSplitChunks(sourceProperties)) {
+ return true;
+ }
+ synchronized (splitsLock) {
+ if (currentOffset != null && !currentOffset.snapshotSplit()) {
+ return true;
+ }
+ return cdcSplitProgress.getCurrentSplittingTable() == null
+ && computeCdcRemainingTables().isEmpty();
+ }
+ }
+
+ /** Tables not yet touched by cdc fetching. Caller must hold splitsLock. */
+ private List<String> computeCdcRemainingTables() {
+ if (cachedSyncTables == null || cachedSyncTables.isEmpty()) {
+ return Collections.emptyList();
+ }
+ // SnapshotSplit.tableId is qualified ("schema.table"/"db.table"); cachedSyncTables is bare — normalize.
+ Set<String> touched = new HashSet<>();
+ for (SnapshotSplit s : finishedSplits) {
+ touched.add(getTableName(s.getTableId()));
+ }
+ for (SnapshotSplit s : remainingSplits) {
+ touched.add(getTableName(s.getTableId()));
+ }
+ if (cdcSplitProgress.getCurrentSplittingTable() != null) {
+ touched.add(getTableName(cdcSplitProgress.getCurrentSplittingTable()));
}
+ List<String> result = new ArrayList<>(cachedSyncTables.size());
+ for (String t : cachedSyncTables) {
+ if (!touched.contains(getTableName(t))) {
+ result.add(t);
+ }
+ }
+ return result;
}
- private void saveChunkMeta(Map<String, List<SnapshotSplit>> tableSplits) throws JobException {
+ @Override
+ public void advanceSplits() throws JobException {
+ // Phase 1 (locked, fast): pick next table & snapshot the resume cursor.
+ String tbl;
+ Object[] startVal;
+ Integer splitId;
+ synchronized (splitsLock) {
+ if (cdcSplitProgress.getCurrentSplittingTable() == null) {
+ List<String> remaining = computeCdcRemainingTables();
+ if (remaining.isEmpty()) {
+ return;
+ }
+ cdcSplitProgress.setCurrentSplittingTable(getTableName(remaining.get(0)));
+ cdcSplitProgress.setNextSplitStart(null);
+ cdcSplitProgress.setNextSplitId(null);
+ }
+ tbl = cdcSplitProgress.getCurrentSplittingTable();
+ startVal = cdcSplitProgress.getNextSplitStart();
+ splitId = cdcSplitProgress.getNextSplitId();
+ }
+
+ // Phase 2 (unlocked, slow): RPC. Keeps updateOffset / scheduler tick unblocked
+ // so task dispatch can drain remainingSplits while we fetch the next batch.
+ List<SnapshotSplit> batch = rpcFetchSplitsBatch(tbl, startVal, splitId);
+ if (batch == null || batch.isEmpty()) {
+ return;
+ }
+
+ // Phase 3 (locked, fast): compute newSplits + splitsOfTbl WITHOUT mutating in-memory.
+ // Persist-before-publish keeps streaming_job_meta from lagging cdcSplitProgress, so a
+ // crash never leaves an HW recorded for a split whose definition was not written.
+ List<SnapshotSplit> newSplits;
+ List<SnapshotSplit> splitsOfTbl;
+ synchronized (splitsLock) {
+ // replayIfNeed / pause-resume may have moved the cursor during the RPC — discard.
+ if (!tbl.equals(cdcSplitProgress.getCurrentSplittingTable())
+ || !Objects.equals(splitId, cdcSplitProgress.getNextSplitId())) {
+ log.info("advanceSplits discard batch for job {} table {}: state moved on "
+ + "during RPC (now table={}, splitId={})",
+ getJobId(), tbl,
+ cdcSplitProgress.getCurrentSplittingTable(),
+ cdcSplitProgress.getNextSplitId());
+ return;
+ }
+ Set<String> existingIds = new HashSet<>();
+ finishedSplits.forEach(s -> existingIds.add(s.getSplitId()));
+ remainingSplits.forEach(s -> existingIds.add(s.getSplitId()));
+ newSplits = new ArrayList<>();
+ for (SnapshotSplit s : batch) {
+ if (!existingIds.contains(s.getSplitId())) {
+ newSplits.add(s);
+ }
+ }
+ if (newSplits.size() < batch.size()) {
+ log.info("advanceSplits dedup'd {} duplicate splits (batch={}, new={}) for job {} table {}",
+ batch.size() - newSplits.size(), batch.size(), newSplits.size(), getJobId(), tbl);
+ }
+ // Post-batch meta state: finished + remaining + newSplits, filtered by table.
+ List<SnapshotSplit> allForTbl = new ArrayList<>(
+ finishedSplits.size() + remainingSplits.size() + newSplits.size());
+ allForTbl.addAll(finishedSplits);
+ allForTbl.addAll(remainingSplits);
+ allForTbl.addAll(newSplits);
+ // tbl is bare (matches cachedSyncTables); SnapshotSplit.tableId is qualified.
+ splitsOfTbl = allForTbl.stream()
+ .filter(s -> tbl.equals(getTableName(s.getTableId())))
+ .sorted(Comparator.comparingInt(s -> splitIdOf(s.getSplitId())))
+ .collect(Collectors.toList());
+ }
+
+ // Phase 4 (unlocked, slow): persist FIRST. On failure throw → PAUSE; autoResume retries
+ // with the same (tbl, startVal, splitId), idempotent on UNIQUE KEY (id, job_id).
try {
- StreamingJobUtils.createMetaTableIfNotExist();
- StreamingJobUtils.insertSplitsToMeta(getJobId(), tableSplits);
+ StreamingJobUtils.upsertChunkList(getJobId(), tbl, splitsOfTbl);
} catch (Exception e) {
- log.warn("save chunk meta error: ", e);
- throw new JobException(e.getMessage());
+ throw new JobException("Failed to persist chunk_list for job " + getJobId()
+ + " table " + tbl + ": " + e.getMessage(), e);
+ }
+
+ // Phase 5 (locked, fast): publish. Skip if cursor moved during Phase 4 (already in meta).
+ synchronized (splitsLock) {
+ if (!tbl.equals(cdcSplitProgress.getCurrentSplittingTable())
+ || !Objects.equals(splitId, cdcSplitProgress.getNextSplitId())) {
+ log.info("advanceSplits discard publish for job {} table {}: state moved on "
+ + "after UPSERT (now table={}, splitId={})",
+ getJobId(), tbl,
+ cdcSplitProgress.getCurrentSplittingTable(),
+ cdcSplitProgress.getNextSplitId());
+ return;
+ }
+ remainingSplits.addAll(newSplits);
+ applySplitToProgress(cdcSplitProgress, batch.get(batch.size() - 1), tbl);
+ log.info("advanceSplits jobId={} table={} request(nextStart={}, nextSplitId={}) "
+ + "published {} new splits, cdcSplitProgress -> (table={}, nextStart={}, nextSplitId={})",
+ getJobId(), tbl, Arrays.toString(startVal), splitId, newSplits.size(),
+ cdcSplitProgress.getCurrentSplittingTable(),
+ Arrays.toString(cdcSplitProgress.getNextSplitStart()),
+ cdcSplitProgress.getNextSplitId());
+ }
+ }
+
+ /** Parse the trailing integer id from flink-cdc splitId format "tableId:id". */
+ static int splitIdOf(String splitId) {
+ if (splitId == null) {
+ throw new IllegalArgumentException("splitId is null");
+ }
+ int colon = splitId.lastIndexOf(':');
+ if (colon < 0 || colon == splitId.length() - 1) {
+ throw new IllegalArgumentException("malformed splitId, expected 'tableId:id': " + splitId);
+ }
+ try {
+ return Integer.parseInt(splitId.substring(colon + 1));
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("malformed splitId, expected 'tableId:id': " + splitId, e);
+ }
+ }
+
+ /** Reset progress to "no table being split" state. */
+ private static void clearProgress(SplitProgress progress) {
+ progress.setCurrentSplittingTable(null);
+ progress.setNextSplitStart(null);
+ progress.setNextSplitId(null);
+ }
+
+ /**
+ * Apply a split's position to a progress object.
+ * - splitEnd null/empty (final split of table) → clear all fields.
+ * - splitEnd non-empty → set currentSplittingTable to tableName (bare, matching the
+ * form used in cachedSyncTables / snapshotTable), advance start/id.
+ * tableName must be the bare name; SnapshotSplit.tableId is qualified (schema.table)
+ * and would break the fetchSplits RPC contract if reused as currentSplittingTable.
+ */
+ private static void applySplitToProgress(SplitProgress progress, SnapshotSplit split, String tableName) {
+ if (split.getSplitEnd() == null || split.getSplitEnd().length == 0) {
+ clearProgress(progress);
+ } else {
+ progress.setCurrentSplittingTable(tableName);
+ progress.setNextSplitStart(split.getSplitEnd());
+ progress.setNextSplitId(splitIdOf(split.getSplitId()) + 1);
}
}
- private List<SnapshotSplit> requestTableSplits(String table) throws JobException {
+ /** RPC fetchSplits with (table, nextSplitStart, nextSplitId, batchSize). protected for UT subclass. */
+ protected List<SnapshotSplit> rpcFetchSplitsBatch(String table, Object[] nextSplitStart, Integer nextSplitId)
+ throws JobException {
Backend backend = StreamingJobUtils.selectBackend(cloudCluster);
- FetchTableSplitsRequest requestParams =
- new FetchTableSplitsRequest(getJobId(), sourceType.name(),
- sourceProperties, getFrontendAddress(), table);
+ FetchTableSplitsRequest requestParams = new FetchTableSplitsRequest(
+ getJobId(), sourceType.name(), sourceProperties, getFrontendAddress(), table);
+ requestParams.setNextSplitStart(nextSplitStart);
+ requestParams.setNextSplitId(nextSplitId);
+ requestParams.setBatchSize(Config.streaming_cdc_fetch_splits_batch_size);
InternalService.PRequestCdcClientRequest request = InternalService.PRequestCdcClientRequest.newBuilder()
.setApi("/api/fetchSplits")
- .setParams(new Gson().toJson(requestParams)).build();
+ .setParams(new Gson().toJson(requestParams))
+ .build();
TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
- InternalService.PRequestCdcClientResult result = null;
try {
Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
.requestCdcClient(address, request, Config.streaming_cdc_heavy_rpc_timeout_sec);
- result = future.get(Config.streaming_cdc_heavy_rpc_timeout_sec, TimeUnit.SECONDS);
+ PRequestCdcClientResult result = future.get(
+ Config.streaming_cdc_heavy_rpc_timeout_sec, TimeUnit.SECONDS);
TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
if (code != TStatusCode.OK) {
- log.warn("Failed to get split from backend, {}", result.getStatus().getErrorMsgs(0));
- throw new JobException(
- "Failed to get split from backend," + result.getStatus().getErrorMsgs(0) + ", response: "
- + result.getResponse());
- }
- String response = result.getResponse();
- try {
- ResponseBody<List<SnapshotSplit>> responseObj = objectMapper.readValue(
- response,
- new TypeReference<ResponseBody<List<SnapshotSplit>>>() {
- }
- );
- List<SnapshotSplit> splits = responseObj.getData();
- return splits;
- } catch (JsonProcessingException e) {
- log.warn("Failed to parse split response: {}", response);
- throw new JobException("Failed to parse split response: " + response);
+ throw new JobException("fetchSplits backend error: " + result.getStatus().getErrorMsgs(0));
}
+ ResponseBody<List<SnapshotSplit>> resp = objectMapper.readValue(
+ result.getResponse(),
+ new TypeReference<ResponseBody<List<SnapshotSplit>>>() {});
+ return resp.getData();
} catch (TimeoutException te) {
- log.warn("cdc_client RPC timeout api=/api/fetchSplits jobId={} backend={}:{} table={} timeout_sec={}",
- getJobId(), backend.getHost(), backend.getBrpcPort(), table,
- Config.streaming_cdc_heavy_rpc_timeout_sec);
- throw new JobException("cdc_client RPC timeout: /api/fetchSplits jobId=" + getJobId() + " table=" + table);
- } catch (ExecutionException | InterruptedException ex) {
- log.warn("Get splits error: ", ex);
- throw new JobException(ex);
+ throw new JobException("fetchSplits RPC timeout: jobId=" + getJobId() + " table=" + table);
+ } catch (Exception ex) {
+ throw new JobException("fetchSplits failed: " + ex.getMessage());
}
}
@@ -673,9 +932,14 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
@Override
public boolean hasReachedEnd() {
- return isSnapshotOnlyMode()
- && CollectionUtils.isNotEmpty(finishedSplits)
- && remainingSplits.isEmpty();
+ if (!isSnapshotOnlyMode()) {
+ return false;
+ }
+ synchronized (splitsLock) {
+ return CollectionUtils.isNotEmpty(finishedSplits)
+ && remainingSplits.isEmpty()
+ && noMoreSplits();
+ }
}
/**
@@ -765,4 +1029,31 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
private String getFrontendAddress() {
return Env.getCurrentEnv().getMasterHost() + ":" + Env.getCurrentEnv().getMasterHttpPort();
}
+
+
+ /** Mirrors flink-cdc ChunkSplitterState. */
+ @Getter
+ @Setter
+ public static class SplitProgress {
+ @SerializedName("ct")
+ private String currentSplittingTable;
+
+ @SerializedName("ns")
+ private Object[] nextSplitStart;
+
+ @SerializedName("ni")
+ private Integer nextSplitId;
+
+ public SplitProgress() {}
+
+ /** Deep copy for transferring committed -> cdc view on restart. */
+ public SplitProgress copy() {
+ SplitProgress c = new SplitProgress();
+ c.currentSplittingTable = this.currentSplittingTable;
+ c.nextSplitStart = this.nextSplitStart == null
+ ? null : Arrays.copyOf(this.nextSplitStart, this.nextSplitStart.length);
+ c.nextSplitId = this.nextSplitId;
+ return c;
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java
index b1d860c9430..9501f62df4f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java
@@ -95,11 +95,7 @@ public class JdbcTvfSourceOffsetProvider extends JdbcSourceOffsetProvider {
super();
}
- /**
- * Initializes provider state and fetches snapshot splits from BE.
- * splitChunks is called here (rather than in StreamingInsertJob) to keep
- * all cdc_stream-specific init logic inside the provider.
- */
+ /** Initializes provider state from TVF properties; called every schedule tick. */
@Override
public void ensureInitialized(Long jobId, Map<String, String> originTvfProps) throws JobException {
String type = originTvfProps.get(DataSourceConfigKeys.TYPE);
@@ -125,16 +121,6 @@ public class JdbcTvfSourceOffsetProvider extends JdbcSourceOffsetProvider {
Preconditions.checkArgument(table != null, "table is required for cdc_stream TVF");
}
- /**
- * Called once on fresh job creation (not on FE restart).
- * Fetches snapshot splits from BE and persists them to the meta table.
- */
- @Override
- public void initOnCreate() throws JobException {
- String table = sourceProperties.get(DataSourceConfigKeys.TABLE);
- splitChunks(Collections.singletonList(table));
- }
-
/**
* Rewrites the cdc_stream TVF SQL with current offset meta and taskId,
* so the BE knows where to start reading and can report
@@ -281,23 +267,25 @@ public class JdbcTvfSourceOffsetProvider extends JdbcSourceOffsetProvider {
public void updateOffset(Offset offset) {
this.currentOffset = (JdbcOffset) offset;
if (currentOffset.snapshotSplit()) {
- for (AbstractSourceSplit split : currentOffset.getSplits()) {
- SnapshotSplit ss = (SnapshotSplit) split;
- boolean removed = remainingSplits.removeIf(v -> {
- if (v.getSplitId().equals(ss.getSplitId())) {
- ss.setTableId(v.getTableId());
- ss.setSplitKey(v.getSplitKey());
- ss.setSplitStart(v.getSplitStart());
- ss.setSplitEnd(v.getSplitEnd());
- return true;
+ synchronized (splitsLock) {
+ for (AbstractSourceSplit split : currentOffset.getSplits()) {
+ SnapshotSplit ss = (SnapshotSplit) split;
+ boolean removed = remainingSplits.removeIf(v -> {
+ if (v.getSplitId().equals(ss.getSplitId())) {
+ ss.setTableId(v.getTableId());
+ ss.setSplitKey(v.getSplitKey());
+ ss.setSplitStart(v.getSplitStart());
+ ss.setSplitEnd(v.getSplitEnd());
+ return true;
+ }
+ return false;
+ });
+ if (removed) {
+ finishedSplits.add(ss);
}
- return false;
- });
- if (removed) {
- finishedSplits.add(ss);
+ chunkHighWatermarkMap.computeIfAbsent(buildTableKey(), k -> new HashMap<>())
+ .put(ss.getSplitId(), ss.getHighWatermark());
}
- chunkHighWatermarkMap.computeIfAbsent(buildTableKey(), k -> new HashMap<>())
- .put(ss.getSplitId(), ss.getHighWatermark());
}
} else {
// Mirror binlog offset into bop so it survives FE checkpoint
@@ -320,6 +308,9 @@ public class JdbcTvfSourceOffsetProvider extends JdbcSourceOffsetProvider {
*/
@Override
public void replayIfNeed(StreamingInsertJob job) throws JobException {
+ synchronized (splitsLock) {
+ this.cachedSyncTables = job.getSyncTables();
+ }
if (currentOffset == null) {
// Post-checkpoint binlog: rebuild from bop persisted in image
if (MapUtils.isNotEmpty(binlogOffsetPersist)) {
@@ -339,6 +330,7 @@ public class JdbcTvfSourceOffsetProvider extends JdbcSourceOffsetProvider {
? remapChunkHighWatermarkMap(snapshotSplits)
: new HashMap<>();
recalculateRemainingSplits(effective, snapshotSplits);
+ resumeCdcSplitProgressFromSplits();
log.info("Replaying TVF offset provider for job {}: no current offset,"
+ " restored {} remaining splits from meta (chw size={})",
job.getJobId(), remainingSplits.size(),
@@ -347,9 +339,7 @@ public class JdbcTvfSourceOffsetProvider extends JdbcSourceOffsetProvider {
log.info("Replaying TVF offset provider for job {}: no committed txn,"
+ " no snapshot splits in meta", job.getJobId());
}
- return;
- }
- if (currentOffset.snapshotSplit()) {
+ } else if (currentOffset.snapshotSplit()) {
log.info("Replaying TVF offset provider for job {}: restoring snapshot state from txn replay",
job.getJobId());
Map<String, List<SnapshotSplit>> snapshotSplits = StreamingJobUtils.restoreSplitsToJob(job.getJobId());
@@ -361,10 +351,12 @@ public class JdbcTvfSourceOffsetProvider extends JdbcSourceOffsetProvider {
remapChunkHighWatermarkMap(snapshotSplits);
List<SnapshotSplit> lastSnapshotSplits =
recalculateRemainingSplits(effectiveMap, snapshotSplits);
+ // Rebuild first so noMoreSplits() can read splitter state from last.splitEnd.
+ resumeCdcSplitProgressFromSplits();
if (remainingSplits.isEmpty()) {
if (!lastSnapshotSplits.isEmpty()) {
currentOffset = new JdbcOffset(lastSnapshotSplits);
- } else if (!isSnapshotOnlyMode()) {
+ } else if (!isSnapshotOnlyMode() && noMoreSplits()) {
BinlogSplit binlogSplit = new BinlogSplit();
binlogSplit.setFinishedSplits(finishedSplits);
currentOffset = new JdbcOffset(Collections.singletonList(binlogSplit));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/util/StreamingJobUtils.java b/fe/fe-core/src/main/java/org/apache/doris/job/util/StreamingJobUtils.java
index 56c95596991..cff663e0142 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/util/StreamingJobUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/util/StreamingJobUtils.java
@@ -101,6 +101,12 @@ public class StreamingJobUtils {
private static final String SELECT_SPLITS_TABLE_TEMPLATE =
"SELECT table_name, chunk_list FROM " + FULL_QUALIFIED_META_TBL_NAME + " WHERE job_id='%s' ORDER BY id ASC";
+ private static final String SELECT_TABLE_ID_TEMPLATE =
+ "SELECT id FROM " + FULL_QUALIFIED_META_TBL_NAME + " WHERE job_id='%s' AND table_name='%s'";
+
+ private static final String SELECT_MAX_ID_TEMPLATE =
+ "SELECT IFNULL(MAX(id), 0) FROM " + FULL_QUALIFIED_META_TBL_NAME + " WHERE job_id='%s'";
+
private static final String DELETE_JOB_META_TEMPLATE =
"DELETE FROM " + FULL_QUALIFIED_META_TBL_NAME + " WHERE job_id='%s'";
@@ -129,6 +135,13 @@ public class StreamingJobUtils {
}
public static Map<String, List<SnapshotSplit>> restoreSplitsToJob(Long jobId) throws JobException {
+ // Meta table is lazy-created on first upsert; skip the internal SQL when absent.
+ Optional<Database> optionalDatabase =
+ Env.getCurrentEnv().getInternalCatalog().getDb(FeConstants.INTERNAL_DB_NAME);
+ if (!optionalDatabase.isPresent()
+ || optionalDatabase.get().getTableNullable(INTERNAL_STREAMING_JOB_META_TABLE_NAME) == null) {
+ return new LinkedHashMap<>();
+ }
List<ResultRow> resultRows;
String sql = String.format(SELECT_SPLITS_TABLE_TEMPLATE, jobId);
try (AutoCloseConnectContext context
@@ -163,21 +176,52 @@ public class StreamingJobUtils {
}
}
- public static void insertSplitsToMeta(Long jobId, Map<String, List<SnapshotSplit>> tableSplits) throws Exception {
- List<String> values = new ArrayList<>();
- int index = 1;
- for (Map.Entry<String, List<SnapshotSplit>> entry : tableSplits.entrySet()) {
- Map<String, String> params = new HashMap<>();
- params.put("id", index + "");
- params.put("job_id", jobId + "");
- params.put("table_name", entry.getKey());
- params.put("chunk_list", objectMapper.writeValueAsString(entry.getValue()));
- StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
- String sql = stringSubstitutor.replace(INSERT_INTO_META_TABLE_TEMPLATE);
- values.add(sql);
- index++;
+ /**
+ * UPSERT a single table's chunk_list. id is reused if the table already has a row,
+ * otherwise allocated as MAX(id)+1. Relies on UNIQUE KEY (id, job_id) for in-place override.
+ */
+ public static void upsertChunkList(Long jobId, String tableName, List<SnapshotSplit> chunks) throws Exception {
+ createMetaTableIfNotExist();
+ Integer id = querySingleTableId(jobId, tableName);
+ if (id == null) {
+ id = queryNextAvailableId(jobId);
+ }
+ Map<String, String> params = new HashMap<>();
+ params.put("id", String.valueOf(id));
+ params.put("job_id", String.valueOf(jobId));
+ params.put("table_name", tableName);
+ params.put("chunk_list", objectMapper.writeValueAsString(chunks));
+ StringSubstitutor sub = new StringSubstitutor(params);
+ String sql = sub.replace(INSERT_INTO_META_TABLE_TEMPLATE);
+ batchInsert(Collections.singletonList(sql));
+ }
+
+ /** Returns id of the row matching (jobId, tableName), or null if no such row exists. */
+ private static Integer querySingleTableId(Long jobId, String tableName) throws JobException {
+ String sql = String.format(SELECT_TABLE_ID_TEMPLATE, jobId, tableName);
+ try (AutoCloseConnectContext ctx = new AutoCloseConnectContext(buildConnectContext())) {
+ StmtExecutor stmtExecutor = new StmtExecutor(ctx.connectContext, sql);
+ List<ResultRow> rows = stmtExecutor.executeInternalQuery();
+ if (rows == null || rows.isEmpty()) {
+ return null;
+ }
+ return Integer.parseInt(rows.get(0).get(0));
+ } catch (Exception e) {
+ throw new JobException("query table id failed: " + e.getMessage());
+ }
+ }
+
+ /** Returns MAX(id) + 1 for this job, or 1 if no rows yet. */
+ private static int queryNextAvailableId(Long jobId) throws JobException {
+ String sql = String.format(SELECT_MAX_ID_TEMPLATE, jobId);
+ try (AutoCloseConnectContext ctx = new AutoCloseConnectContext(buildConnectContext())) {
+ StmtExecutor stmtExecutor = new StmtExecutor(ctx.connectContext, sql);
+ List<ResultRow> rows = stmtExecutor.executeInternalQuery();
+ int max = (rows == null || rows.isEmpty()) ? 0 : Integer.parseInt(rows.get(0).get(0));
+ return max + 1;
+ } catch (Exception e) {
+ throw new JobException("query next available id failed: " + e.getMessage());
}
- batchInsert(values);
}
private static void batchInsert(List<String> values) throws Exception {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProviderAsyncSplitTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProviderAsyncSplitTest.java
new file mode 100644
index 00000000000..e6cd257ffee
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProviderAsyncSplitTest.java
@@ -0,0 +1,486 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.job.offset.jdbc;
+
+import org.apache.doris.job.cdc.split.BinlogSplit;
+import org.apache.doris.job.cdc.split.SnapshotSplit;
+import org.apache.doris.job.exception.JobException;
+import org.apache.doris.job.util.StreamingJobUtils;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentMatchers;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.List;
+
+/**
+ * Tests the async split state machine in {@link JdbcSourceOffsetProvider}:
+ * advanceSplits / noMoreSplits / updateOffset / dedup. RPC and the system-table
+ * UPSERT are stubbed so the test focuses purely on in-memory state transitions.
+ */
+public class JdbcSourceOffsetProviderAsyncSplitTest {
+
+ /** Records each rpcFetchSplitsBatch invocation; used for argument assertions. */
+ static final class RpcCall {
+ final String table;
+ final Object[] startVal;
+ final Integer splitId;
+
+ RpcCall(String table, Object[] startVal, Integer splitId) {
+ this.table = table;
+ this.startVal = startVal;
+ this.splitId = splitId;
+ }
+ }
+
+ /** Provider under test with rpcFetchSplitsBatch stubbed to dequeue prepared batches. */
+ static final class TestableProvider extends JdbcSourceOffsetProvider {
+ final Deque<List<SnapshotSplit>> mockBatches = new ArrayDeque<>();
+ final List<RpcCall> rpcCalls = new ArrayList<>();
+
+ TestableProvider() {
+ super();
+ // Default to initial mode so initOnCreate() takes the splitting path
+ // (latest mode would try to call initSourceReader against a real backend).
+ this.sourceProperties.put(
+ org.apache.doris.job.cdc.DataSourceConfigKeys.OFFSET,
+ org.apache.doris.job.cdc.DataSourceConfigKeys.OFFSET_INITIAL);
+ }
+
+ @Override
+ protected List<SnapshotSplit> rpcFetchSplitsBatch(String table, Object[] startVal, Integer splitId) {
+ rpcCalls.add(new RpcCall(table, startVal, splitId));
+ if (mockBatches.isEmpty()) {
+ return Collections.emptyList();
+ }
+ return mockBatches.poll();
+ }
+
+ @Override
+ public Long getJobId() {
+ return 999L;
+ }
+ }
+
+ private TestableProvider provider;
+ private MockedStatic<StreamingJobUtils> utilsMock;
+
+ @Before
+ public void setup() {
+ provider = new TestableProvider();
+ utilsMock = Mockito.mockStatic(StreamingJobUtils.class);
+ utilsMock.when(() -> StreamingJobUtils.upsertChunkList(
+ ArgumentMatchers.anyLong(),
+ ArgumentMatchers.anyString(),
+ ArgumentMatchers.any()))
+ .then(invocation -> null);
+ }
+
+ @org.junit.After
+ public void tearDown() {
+ if (utilsMock != null) {
+ utilsMock.close();
+ }
+ }
+
+ /** Helper to build a SnapshotSplit. start/end are wrapped in Object[] only when non-null. */
+ private static SnapshotSplit split(String tableId, int chunkId, Long start, Long end) {
+ return new SnapshotSplit(
+ tableId + ":" + chunkId,
+ tableId,
+ Collections.singletonList("id"),
+ start == null ? null : new Object[]{start},
+ end == null ? null : new Object[]{end},
+ Collections.singletonMap("file", "binlog.000001"));
+ }
+
+ // ===== initOnCreate / noMoreSplits =====
+
+ @Test
+ public void testInitWithEmptySyncTablesIsAllDone() throws JobException {
+ provider.initOnCreate(Collections.emptyList());
+ Assert.assertTrue(provider.noMoreSplits());
+ }
+
+ @Test
+ public void testInitWithSyncTablesNotDone() throws JobException {
+ provider.initOnCreate(Arrays.asList("db.tbl_a"));
+ Assert.assertNotNull(provider.committedSplitProgress);
+ Assert.assertNotNull(provider.cdcSplitProgress);
+ Assert.assertNull(provider.cdcSplitProgress.getCurrentSplittingTable());
+ Assert.assertFalse(provider.noMoreSplits());
+ }
+
+ @Test
+ public void testNoMoreSplitsTrueAfterBinlogTransition() throws JobException {
+ // Mirrors restart-via-binlogOffsetPersist replay path: currentOffset=BinlogSplit
+ // with finishedSplits/remainingSplits empty. Without the BinlogSplit fast-path
+ // computeCdcRemainingTables would see every syncTable as untouched and let the
+ // scheduler re-cut snapshot chunks after snapshot phase is over.
+ provider.initOnCreate(Arrays.asList("db.tbl_a", "db.tbl_b"));
+ provider.setCurrentOffset(new JdbcOffset(Collections.singletonList(new BinlogSplit())));
+ Assert.assertTrue(provider.noMoreSplits());
+ }
+
+ @Test
+ public void testNoMoreSplitsStillFalseDuringSnapshot() throws JobException {
+ // Safety regression: currentOffset=snapshotSplit + untouched table -> fast-path
+ // must NOT fire (otherwise advanceSplits would never be called).
+ provider.initOnCreate(Arrays.asList("db.tbl_a"));
+ provider.setCurrentOffset(new JdbcOffset(
+ Collections.singletonList(split("db.tbl_a", 0, null, 100L))));
+ Assert.assertFalse(provider.noMoreSplits());
+ }
+
+ // ===== advanceSplits =====
+
+ @Test
+ public void testAdvanceFirstCallPicksFirstTableWithNullStart() throws JobException {
+ provider.initOnCreate(Arrays.asList("db.tbl_a", "db.tbl_b"));
+ provider.mockBatches.add(Arrays.asList(
+ split("db.tbl_a", 0, null, 100L),
+ split("db.tbl_a", 1, 100L, 200L)));
+
+ provider.advanceSplits();
+
+ Assert.assertEquals(2, provider.remainingSplits.size());
+ Assert.assertEquals("tbl_a", provider.cdcSplitProgress.getCurrentSplittingTable());
+ Assert.assertArrayEquals(new Object[]{200L}, provider.cdcSplitProgress.getNextSplitStart());
+ Assert.assertEquals(Integer.valueOf(2), provider.cdcSplitProgress.getNextSplitId());
+
+ Assert.assertEquals(1, provider.rpcCalls.size());
+ RpcCall first = provider.rpcCalls.get(0);
+ Assert.assertEquals("tbl_a", first.table);
+ Assert.assertNull("first call should pass null nextSplitStart (= START_BOUND)", first.startVal);
+ Assert.assertNull(first.splitId);
+ }
+
+ @Test
+ public void testAdvanceContinuesOnSameTableAfterFirstBatch() throws JobException {
+ provider.initOnCreate(Arrays.asList("db.tbl_a"));
+ provider.mockBatches.add(Arrays.asList(split("db.tbl_a", 0, null, 100L)));
+ provider.mockBatches.add(Arrays.asList(split("db.tbl_a", 1, 100L, 200L)));
+
+ provider.advanceSplits();
+ provider.advanceSplits();
+
+ Assert.assertEquals(2, provider.rpcCalls.size());
+ RpcCall second = provider.rpcCalls.get(1);
+ Assert.assertEquals("tbl_a", second.table);
+ Assert.assertArrayEquals(new Object[]{100L}, second.startVal);
+ Assert.assertEquals(Integer.valueOf(1), second.splitId);
+ Assert.assertEquals(2, provider.remainingSplits.size());
+ }
+
+ @Test
+ public void testAdvanceTableDoneSwitchesToNextTable() throws JobException {
+ provider.initOnCreate(Arrays.asList("db.tbl_a", "db.tbl_b"));
+ // tbl_a's last chunk: splitEnd=null
+ provider.mockBatches.add(Arrays.asList(split("db.tbl_a", 0, null, null)));
+ // 2nd advance picks tbl_b
+ provider.mockBatches.add(Arrays.asList(split("db.tbl_b", 0, null, 50L)));
+
+ provider.advanceSplits();
+ Assert.assertNull("after tbl_a done, currentSplittingTable should clear",
+ provider.cdcSplitProgress.getCurrentSplittingTable());
+ Assert.assertFalse("tbl_b still pending", provider.noMoreSplits());
+
+ provider.advanceSplits();
+
+ Assert.assertEquals(2, provider.rpcCalls.size());
+ Assert.assertEquals("tbl_b", provider.rpcCalls.get(1).table);
+ Assert.assertEquals("tbl_b", provider.cdcSplitProgress.getCurrentSplittingTable());
+ Assert.assertArrayEquals(new Object[]{50L}, provider.cdcSplitProgress.getNextSplitStart());
+ }
+
+ @Test
+ public void testAllSyncTablesDoneMakesNoMoreSplitsTrue() throws JobException {
+ provider.initOnCreate(Arrays.asList("db.tbl_a"));
+ provider.mockBatches.add(Arrays.asList(split("db.tbl_a", 0, null, null)));
+
+ provider.advanceSplits();
+
+ Assert.assertTrue(provider.noMoreSplits());
+ Assert.assertEquals(1, provider.remainingSplits.size());
+ }
+
+ @Test
+ public void testAdvanceSplitsDedupsBySplitId() throws JobException {
+ provider.initOnCreate(Arrays.asList("db.tbl_a"));
+ // Pre-existing split with same splitId; simulates a defensive dedup target
+ // (e.g. on FE restart after RPC succeeded but state wasn't fully advanced).
+ provider.remainingSplits.add(split("db.tbl_a", 0, null, 100L));
+ provider.mockBatches.add(Arrays.asList(split("db.tbl_a", 0, null, 100L)));
+
+ provider.advanceSplits();
+
+ Assert.assertEquals("duplicate splitId should be filtered out",
+ 1, provider.remainingSplits.size());
+ }
+
+ @Test
+ public void testAdvanceWithEmptyBatchIsNoop() throws JobException {
+ provider.initOnCreate(Arrays.asList("db.tbl_a"));
+ // mockBatches empty → rpcFetchSplitsBatch returns empty list
+ provider.advanceSplits();
+
+ Assert.assertEquals(0, provider.remainingSplits.size());
+ // currentSplittingTable was set then RPC returned empty; we leave it set
+ // (next advance retries on same table from null start). Just assert no progress.
+ Assert.assertNull(provider.cdcSplitProgress.getNextSplitStart());
+ Assert.assertNull(provider.cdcSplitProgress.getNextSplitId());
+ }
+
+ // ===== updateOffset advances committedSplitProgress =====
+
+ /** Build a commit-shaped SnapshotSplit: only splitId + HW are present (BE strips others). */
+ private static SnapshotSplit commitSplit(String splitId) {
+ SnapshotSplit s = new SnapshotSplit();
+ s.setSplitId(splitId);
+ s.setHighWatermark(Collections.singletonMap("file", "binlog.000002"));
+ return s;
+ }
+
+ @Test
+ public void testUpdateOffsetAdvancesCommittedProgressOnMidChunk() throws JobException {
+ provider.initOnCreate(Arrays.asList("db.tbl_a"));
+ provider.mockBatches.add(Arrays.asList(
+ split("db.tbl_a", 0, null, 100L),
+ split("db.tbl_a", 1, 100L, 200L)));
+ provider.advanceSplits();
+
+ // Task commits chunk #0; updateOffset will copy splitEnd back from remainingSplits.
+ JdbcOffset endOffset = new JdbcOffset(Collections.singletonList(commitSplit("db.tbl_a:0")));
+ provider.updateOffset(endOffset);
+
+ Assert.assertEquals(1, provider.finishedSplits.size());
+ Assert.assertEquals(1, provider.remainingSplits.size());
+
+ JdbcSourceOffsetProvider.SplitProgress committed = provider.committedSplitProgress;
+ Assert.assertEquals("tbl_a", committed.getCurrentSplittingTable());
+ Assert.assertArrayEquals(new Object[]{100L}, committed.getNextSplitStart());
+ Assert.assertEquals(Integer.valueOf(1), committed.getNextSplitId());
+ }
+
+ @Test
+ public void testUpdateOffsetLastChunkClearsCommittedProgress() throws JobException {
+ provider.initOnCreate(Arrays.asList("db.tbl_a"));
+ provider.mockBatches.add(Arrays.asList(split("db.tbl_a", 0, null, null)));
+ provider.advanceSplits();
+
+ JdbcOffset endOffset = new JdbcOffset(Collections.singletonList(commitSplit("db.tbl_a:0")));
+ provider.updateOffset(endOffset);
+
+ JdbcSourceOffsetProvider.SplitProgress committed = provider.committedSplitProgress;
+ Assert.assertNull(committed.getCurrentSplittingTable());
+ Assert.assertNull(committed.getNextSplitStart());
+ Assert.assertNull(committed.getNextSplitId());
+ Assert.assertEquals(1, provider.finishedSplits.size());
+ Assert.assertEquals(0, provider.remainingSplits.size());
+ }
+
+ @Test
+ public void testUpdateOffsetReplayPathSkipsWhenSplitMissing() throws JobException {
+ provider.initOnCreate(Arrays.asList("db.tbl_a"));
+ // remainingSplits is empty (simulates editlog replay path).
+
+ JdbcOffset endOffset = new JdbcOffset(Collections.singletonList(commitSplit("db.tbl_a:0")));
+ provider.updateOffset(endOffset);
+
+ // committed progress untouched; finishedSplits not added (we have nothing to fill in).
+ Assert.assertNull(provider.committedSplitProgress.getCurrentSplittingTable());
+ Assert.assertEquals(0, provider.finishedSplits.size());
+ }
+
+ // ===== computeCdcRemainingTables (covered indirectly via noMoreSplits) =====
+
+ @Test
+ public void testTouchedTablesRemovedFromRemaining() throws JobException {
+ provider.initOnCreate(Arrays.asList("db.tbl_a", "db.tbl_b", "db.tbl_c"));
+ provider.mockBatches.add(Arrays.asList(split("db.tbl_a", 0, null, null)));
+ provider.advanceSplits();
+
+ // tbl_a is now done (in remainingSplits + currentSplittingTable cleared).
+ // 2 more tables remain; noMoreSplits should still be false.
+ Assert.assertFalse(provider.noMoreSplits());
+ Assert.assertNull(provider.cdcSplitProgress.getCurrentSplittingTable());
+
+ // 2nd advance picks tbl_b
+ provider.mockBatches.add(Arrays.asList(split("db.tbl_b", 0, null, null)));
+ provider.advanceSplits();
+ Assert.assertEquals("tbl_b", provider.rpcCalls.get(1).table);
+
+ // 3rd advance picks tbl_c
+ provider.mockBatches.add(Arrays.asList(split("db.tbl_c", 0, null, null)));
+ provider.advanceSplits();
+ Assert.assertEquals("tbl_c", provider.rpcCalls.get(2).table);
+
+ Assert.assertTrue(provider.noMoreSplits());
+ }
+
+ // ===== findResumeMidSplit (replay helper) =====
+
+ @Test
+ public void testFindResumeMidSplitSingleTableFullyCutReturnsNull() {
+ SnapshotSplit s0 = split("db.tbl_a", 0, null, 100L);
+ SnapshotSplit s1 = split("db.tbl_a", 1, 100L, null); // last, splitEnd=null
+ SnapshotSplit mid = JdbcSourceOffsetProvider.findResumeMidSplit(
+ Collections.singletonList("db.tbl_a"),
+ Arrays.asList(s0, s1),
+ Collections.emptyList());
+ Assert.assertNull(mid);
+ }
+
+ @Test
+ public void testFindResumeMidSplitSingleTableCutToMid() {
+ SnapshotSplit s0 = split("db.tbl_a", 0, null, 100L);
+ SnapshotSplit s1 = split("db.tbl_a", 1, 100L, 200L); // largest id, splitEnd non-null
+ SnapshotSplit mid = JdbcSourceOffsetProvider.findResumeMidSplit(
+ Collections.singletonList("db.tbl_a"),
+ Arrays.asList(s0, s1),
+ Collections.emptyList());
+ Assert.assertNotNull(mid);
+ Assert.assertEquals("db.tbl_a:1", mid.getSplitId());
+ Assert.assertArrayEquals(new Object[]{200L}, mid.getSplitEnd());
+ }
+
+ @Test
+ public void testFindResumeMidSplitMultiTableOnlyOneMid() {
+ // tbl_a fully cut; tbl_b cut to mid; tbl_c untouched
+ SnapshotSplit a0 = split("db.tbl_a", 0, null, null);
+ SnapshotSplit b0 = split("db.tbl_b", 0, null, 50L);
+ SnapshotSplit mid = JdbcSourceOffsetProvider.findResumeMidSplit(
+ Arrays.asList("db.tbl_a", "db.tbl_b", "db.tbl_c"),
+ Collections.singletonList(a0),
+ Collections.singletonList(b0));
+ Assert.assertNotNull(mid);
+ Assert.assertEquals("db.tbl_b:0", mid.getSplitId());
+ }
+
+ @Test
+ public void testFindResumeMidSplitMaxIdSpreadAcrossLists() {
+ // last id is in remainingSplits (id=2), not finishedSplits (id=0,1)
+ SnapshotSplit f0 = split("db.tbl_a", 0, null, 100L);
+ SnapshotSplit f1 = split("db.tbl_a", 1, 100L, 200L);
+ SnapshotSplit r2 = split("db.tbl_a", 2, 200L, 300L);
+ SnapshotSplit mid = JdbcSourceOffsetProvider.findResumeMidSplit(
+ Collections.singletonList("db.tbl_a"),
+ Arrays.asList(f0, f1),
+ Collections.singletonList(r2));
+ Assert.assertNotNull(mid);
+ Assert.assertEquals("db.tbl_a:2", mid.getSplitId());
+ Assert.assertArrayEquals(new Object[]{300L}, mid.getSplitEnd());
+ }
+
+ @Test
+ public void testFindResumeMidSplitEmptyInputs() {
+ Assert.assertNull(JdbcSourceOffsetProvider.findResumeMidSplit(
+ Collections.emptyList(), Collections.emptyList(), Collections.emptyList()));
+ Assert.assertNull(JdbcSourceOffsetProvider.findResumeMidSplit(
+ Collections.singletonList("db.tbl_a"),
+ Collections.emptyList(), Collections.emptyList()));
+ }
+
+ @Test
+ public void testFindResumeMidSplitBareSyncTableQualifiedSplitTableId() {
+ // Production layout: cachedSyncTables = bare ("tbl_a"); SnapshotSplit.tableId = qualified
+ // ("schema.tbl_a"). Map keys must normalize to bare on both sides, otherwise the lookup
+ // misses and resume returns null even when there is a mid-cut to continue from.
+ SnapshotSplit s0 = split("schema.tbl_a", 0, null, 100L);
+ SnapshotSplit s1 = split("schema.tbl_a", 1, 100L, 200L);
+ SnapshotSplit mid = JdbcSourceOffsetProvider.findResumeMidSplit(
+ Collections.singletonList("tbl_a"),
+ Arrays.asList(s0, s1),
+ Collections.emptyList());
+ Assert.assertNotNull(mid);
+ Assert.assertEquals("schema.tbl_a:1", mid.getSplitId());
+ }
+
+ @Test
+ public void testFindResumeMidSplitSyncTablesContainsUntouchedTable() {
+ // syncTables lists tbl_a and tbl_b; only tbl_a appears in splits, fully cut.
+ // tbl_b is untouched (no splits) -> still returns null (no mid).
+ SnapshotSplit a0 = split("db.tbl_a", 0, null, null);
+ SnapshotSplit mid = JdbcSourceOffsetProvider.findResumeMidSplit(
+ Arrays.asList("db.tbl_a", "db.tbl_b"),
+ Collections.singletonList(a0),
+ Collections.emptyList());
+ Assert.assertNull(mid);
+ }
+
+ // ===== splitIdOf validation =====
+
+ @Test
+ public void testSplitIdOfHappyPath() {
+ Assert.assertEquals(0, JdbcSourceOffsetProvider.splitIdOf("db.tbl_a:0"));
+ Assert.assertEquals(42, JdbcSourceOffsetProvider.splitIdOf("db.tbl_a:42"));
+ // table with colon in its qualifier: lastIndexOf(':') takes the trailing one.
+ Assert.assertEquals(7, JdbcSourceOffsetProvider.splitIdOf("schema:tbl:7"));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testSplitIdOfNoColonThrows() {
+ JdbcSourceOffsetProvider.splitIdOf("db.tbl_a_0");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testSplitIdOfTrailingColonThrows() {
+ JdbcSourceOffsetProvider.splitIdOf("db.tbl_a:");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testSplitIdOfNonNumericSuffixThrows() {
+ JdbcSourceOffsetProvider.splitIdOf("db.tbl_a:abc");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testSplitIdOfNullThrows() {
+ JdbcSourceOffsetProvider.splitIdOf(null);
+ }
+
+ // ===== mode gate =====
+
+ @Test
+ public void testNoMoreSplitsLatestModeAlwaysTrue() {
+ provider.sourceProperties.put(
+ org.apache.doris.job.cdc.DataSourceConfigKeys.OFFSET,
+ org.apache.doris.job.cdc.DataSourceConfigKeys.OFFSET_LATEST);
+ // Even if cachedSyncTables is populated (e.g. by replayIfNeed), latest mode
+ // must report noMoreSplits=true so scheduler skips advanceSplits entirely.
+ provider.cachedSyncTables = Arrays.asList("db.tbl_a", "db.tbl_b");
+ Assert.assertTrue(provider.noMoreSplits());
+ }
+
+ @Test
+ public void testNoMoreSplitsSnapshotModeStillRespectsState() throws JobException {
+ provider.sourceProperties.put(
+ org.apache.doris.job.cdc.DataSourceConfigKeys.OFFSET,
+ org.apache.doris.job.cdc.DataSourceConfigKeys.OFFSET_SNAPSHOT);
+ provider.initOnCreate(Arrays.asList("db.tbl_a"));
+ Assert.assertFalse("snapshot mode with un-split tables must return false",
+ provider.noMoreSplits());
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/SplitProgressTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/SplitProgressTest.java
new file mode 100644
index 00000000000..9fccdc0b012
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/SplitProgressTest.java
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.job.offset.jdbc;
+
+import org.apache.doris.job.offset.jdbc.JdbcSourceOffsetProvider.SplitProgress;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class SplitProgressTest {
+
+ @Test
+ public void testDefaultStateIsAllNull() {
+ SplitProgress p = new SplitProgress();
+ Assert.assertNull(p.getCurrentSplittingTable());
+ Assert.assertNull(p.getNextSplitStart());
+ Assert.assertNull(p.getNextSplitId());
+ }
+
+ @Test
+ public void testCopyDeepClonesNextSplitStart() {
+ SplitProgress original = new SplitProgress();
+ original.setCurrentSplittingTable("db.tbl_a");
+ original.setNextSplitStart(new Object[]{100L});
+ original.setNextSplitId(5);
+
+ SplitProgress copy = original.copy();
+ Assert.assertEquals("db.tbl_a", copy.getCurrentSplittingTable());
+ Assert.assertArrayEquals(new Object[]{100L}, copy.getNextSplitStart());
+ Assert.assertEquals(Integer.valueOf(5), copy.getNextSplitId());
+
+ // Mutating copy.nextSplitStart must not affect the original (deep copy).
+ copy.getNextSplitStart()[0] = 999L;
+ Assert.assertEquals(100L, original.getNextSplitStart()[0]);
+ }
+
+ @Test
+ public void testCopyHandlesNullFields() {
+ SplitProgress original = new SplitProgress();
+ SplitProgress copy = original.copy();
+ Assert.assertNull(copy.getCurrentSplittingTable());
+ Assert.assertNull(copy.getNextSplitStart());
+ Assert.assertNull(copy.getNextSplitId());
+ }
+}
diff --git a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/JdbcIncrementalSourceReader.java b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/JdbcIncrementalSourceReader.java
index ae64b695e95..826866e5885 100644
--- a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/JdbcIncrementalSourceReader.java
+++ b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/JdbcIncrementalSourceReader.java
@@ -30,14 +30,14 @@ import org.apache.doris.job.cdc.split.SnapshotSplit;
import org.apache.commons.collections.CollectionUtils;
import org.apache.flink.api.connector.source.SourceSplit;
-import org.apache.flink.api.connector.source.mocks.MockSplitEnumeratorContext;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.cdc.common.utils.Preconditions;
import org.apache.flink.cdc.connectors.base.config.JdbcSourceConfig;
import org.apache.flink.cdc.connectors.base.dialect.JdbcDataSourceDialect;
import org.apache.flink.cdc.connectors.base.options.StartupOptions;
-import org.apache.flink.cdc.connectors.base.source.assigner.HybridSplitAssigner;
-import org.apache.flink.cdc.connectors.base.source.assigner.SnapshotSplitAssigner;
+import org.apache.flink.cdc.connectors.base.source.assigner.splitter.ChunkSplitter;
+import org.apache.flink.cdc.connectors.base.source.assigner.state.ChunkSplitterState;
+import org.apache.flink.cdc.connectors.base.source.assigner.state.ChunkSplitterState.ChunkBound;
import org.apache.flink.cdc.connectors.base.source.meta.offset.Offset;
import org.apache.flink.cdc.connectors.base.source.meta.offset.OffsetFactory;
import org.apache.flink.cdc.connectors.base.source.meta.split.FinishedSnapshotSplitInfo;
@@ -50,6 +50,7 @@ import org.apache.flink.cdc.connectors.base.source.meta.split.StreamSplitState;
import org.apache.flink.cdc.connectors.base.source.meta.wartermark.WatermarkEvent;
import org.apache.flink.cdc.connectors.base.source.reader.external.FetchTask;
import org.apache.flink.cdc.connectors.base.source.reader.external.Fetcher;
+import org.apache.flink.cdc.connectors.base.source.utils.JdbcChunkUtils;
import org.apache.flink.cdc.connectors.base.utils.SourceRecordUtils;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.types.DataType;
@@ -58,6 +59,7 @@ import org.apache.kafka.connect.source.SourceRecord;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -66,7 +68,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
-import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
@@ -133,54 +134,95 @@ public abstract class JdbcIncrementalSourceReader extends AbstractCdcSourceReade
LOG.info("Initialized poll executor with parallelism: {}", parallelism);
}
+ /**
+ * Fetch a batch of snapshot splits by driving flink-cdc {@link ChunkSplitter} directly.
+ *
+ * <p>Stateless: each RPC builds a fresh splitter from the (table, nextChunkStart, nextChunkId)
+ * triple supplied by FE, fetches up to {@code batchSize} chunks, then closes the splitter.
+ *
+ * <p>Only INITIAL/SNAPSHOT startup modes call this RPC; binlog/latest modes never reach here.
+ */
@Override
public List<AbstractSourceSplit> getSourceSplits(FetchTableSplitsRequest ftsReq) {
- LOG.info("Get table {} splits for job {}", ftsReq.getSnapshotTable(), ftsReq.getJobId());
+ LOG.info(
+ "Get table {} splits for job {} (nextSplitId={}, nextSplitStart={})",
+ ftsReq.getSnapshotTable(),
+ ftsReq.getJobId(),
+ ftsReq.getNextSplitId(),
+ java.util.Arrays.toString(ftsReq.getNextSplitStart()));
JdbcSourceConfig sourceConfig = getSourceConfig(ftsReq);
- List<org.apache.flink.cdc.connectors.base.source.meta.split.SnapshotSplit>
- remainingSnapshotSplits = new ArrayList<>();
- StreamSplit remainingStreamSplit = null;
-
- // Check startup mode - for PostgreSQL, we use similar logic as MySQL
- String startupMode = ftsReq.getConfig().get(DataSourceConfigKeys.OFFSET);
- if (DataSourceConfigKeys.OFFSET_INITIAL.equalsIgnoreCase(startupMode)
- || DataSourceConfigKeys.OFFSET_SNAPSHOT.equalsIgnoreCase(startupMode)) {
- remainingSnapshotSplits =
- startSplitChunks(sourceConfig, ftsReq.getSnapshotTable(), ftsReq.getConfig());
- } else {
- // For non-initial mode, create a stream split
- Offset startingOffset = createInitialOffset();
- remainingStreamSplit =
- new StreamSplit(
- STREAM_SPLIT_ID,
- startingOffset,
- createNoStoppingOffset(),
- new ArrayList<>(),
- new HashMap<>(),
- 0);
- }
+ String schema = ftsReq.getConfig().get(DataSourceConfigKeys.SCHEMA);
+ TableId tableId = new TableId(null, schema, ftsReq.getSnapshotTable());
+
+ int batchSize = ftsReq.getBatchSize() == null ? 100 : ftsReq.getBatchSize();
+ ChunkSplitterState state = buildChunkSplitterState(sourceConfig, tableId, ftsReq);
+ ChunkSplitter splitter = getDialect(sourceConfig).createChunkSplitter(sourceConfig, state);
- List<AbstractSourceSplit> splits = new ArrayList<>();
- if (!remainingSnapshotSplits.isEmpty()) {
- for (org.apache.flink.cdc.connectors.base.source.meta.split.SnapshotSplit
- snapshotSplit : remainingSnapshotSplits) {
- String splitId = snapshotSplit.splitId();
- String tableId = snapshotSplit.getTableId().identifier();
- Object[] splitStart = snapshotSplit.getSplitStart();
- Object[] splitEnd = snapshotSplit.getSplitEnd();
- List<String> splitKey = snapshotSplit.getSplitKeyType().getFieldNames();
- SnapshotSplit split =
- new SnapshotSplit(splitId, tableId, splitKey, splitStart, splitEnd, null);
- splits.add(split);
+ try {
+ splitter.open();
+ List<AbstractSourceSplit> result = new ArrayList<>();
+ while (result.size() < batchSize) {
+ Collection<org.apache.flink.cdc.connectors.base.source.meta.split.SnapshotSplit>
+ chunks = splitter.generateSplits(tableId);
+ for (org.apache.flink.cdc.connectors.base.source.meta.split.SnapshotSplit chunk :
+ chunks) {
+ result.add(toDorisSnapshotSplit(chunk));
+ }
+ if (!splitter.hasNextChunk()) {
+ break;
+ }
}
- } else {
- Offset startingOffset = remainingStreamSplit.getStartingOffset();
- BinlogSplit streamSplit = new BinlogSplit();
- streamSplit.setSplitId(remainingStreamSplit.splitId());
- streamSplit.setStartingOffset(startingOffset.getOffset());
- splits.add(streamSplit);
+ LOG.info(
+ "Fetched {} splits for table {} (resume nextSplitId={}); hasNextChunk={}",
+ result.size(),
+ tableId,
+ ftsReq.getNextSplitId(),
+ splitter.hasNextChunk());
+ return result;
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to generate splits for " + tableId, e);
+ } finally {
+ try {
+ splitter.close();
+ } catch (Exception e) {
+ LOG.warn("Failed to close splitter for {}", tableId, e);
+ }
+ }
+ }
+
+ /**
+ * null start -> NO_SPLITTING_TABLE_STATE (analyze + maybe evenly); non-null -> resume
+ * mid-table. Cast pkValues[0] back to the JDBC driver's natural type (JSON round-trip
+ * downgrades types).
+ */
+ private ChunkSplitterState buildChunkSplitterState(
+ JdbcSourceConfig sourceConfig, TableId tableId, FetchTableSplitsRequest ftsReq) {
+ Object[] pkValues = ftsReq.getNextSplitStart();
+ if (pkValues == null || pkValues.length == 0) {
+ return ChunkSplitterState.NO_SPLITTING_TABLE_STATE;
}
- return splits;
+ TableChanges.TableChange tableChange = getTableSchemas(ftsReq).get(tableId);
+ Column splitColumn =
+ JdbcChunkUtils.getSplitColumn(
+ tableChange.getTable(), sourceConfig.getChunkKeyColumn());
+ Class<?> targetClass = resolveSplitKeyClass(tableId, splitColumn, ftsReq);
+ Object[] castStart = convertBounds(pkValues, targetClass, objectMapper);
+ int splitId = ftsReq.getNextSplitId() == null ? 0 : ftsReq.getNextSplitId();
+ return new ChunkSplitterState(tableId, ChunkBound.middleOf(castStart[0]), splitId);
+ }
+
+ /**
+ * flink-cdc SnapshotSplit -> Doris SnapshotSplit (drops splitKeyType, keeps key field names).
+ */
+ private SnapshotSplit toDorisSnapshotSplit(
+ org.apache.flink.cdc.connectors.base.source.meta.split.SnapshotSplit chunk) {
+ return new SnapshotSplit(
+ chunk.splitId(),
+ chunk.getTableId().identifier(),
+ chunk.getSplitKeyType().getFieldNames(),
+ chunk.getSplitStart(),
+ chunk.getSplitEnd(),
+ null);
}
@Override
@@ -773,86 +815,6 @@ public abstract class JdbcIncrementalSourceReader extends AbstractCdcSourceReade
return startingOffset;
}
- private List<org.apache.flink.cdc.connectors.base.source.meta.split.SnapshotSplit>
- startSplitChunks(
- JdbcSourceConfig sourceConfig,
- String snapshotTable,
- Map<String, String> config) {
- List<TableId> remainingTables = new ArrayList<>();
- if (snapshotTable != null) {
- String schema = config.get(DataSourceConfigKeys.SCHEMA);
- remainingTables.add(new TableId(null, schema, snapshotTable));
- }
- List<org.apache.flink.cdc.connectors.base.source.meta.split.SnapshotSplit> remainingSplits =
- new ArrayList<>();
- HybridSplitAssigner<JdbcSourceConfig> splitAssigner =
- new HybridSplitAssigner<>(
- sourceConfig,
- 1,
- remainingTables,
- true,
- getDialect(sourceConfig),
- getOffsetFactory(),
- new MockSplitEnumeratorContext(1));
- splitAssigner.open();
- try {
- while (true) {
- Optional<SourceSplitBase> split = splitAssigner.getNext();
- if (split.isPresent()) {
- org.apache.flink.cdc.connectors.base.source.meta.split.SnapshotSplit
- snapshotSplit = split.get().asSnapshotSplit();
- remainingSplits.add(snapshotSplit);
- } else {
- break;
- }
- }
- } finally {
- closeChunkSplitterOnly(splitAssigner);
- }
- return remainingSplits;
- }
-
- /**
- * Close only the chunk splitter to avoid closing shared connection pools Similar to MySQL
- * implementation Note: HybridSplitAssigner wraps SnapshotSplitAssigner, so we need to get the
- * inner assigner first
- */
- private static void closeChunkSplitterOnly(HybridSplitAssigner<?> splitAssigner) {
- try {
- // First, get the inner SnapshotSplitAssigner from HybridSplitAssigner
- java.lang.reflect.Field snapshotAssignerField =
- HybridSplitAssigner.class.getDeclaredField("snapshotSplitAssigner");
- snapshotAssignerField.setAccessible(true);
- SnapshotSplitAssigner<?> snapshotSplitAssigner =
- (SnapshotSplitAssigner<?>) snapshotAssignerField.get(splitAssigner);
-
- if (snapshotSplitAssigner == null) {
- LOG.warn("snapshotSplitAssigner is null in HybridSplitAssigner");
- return;
- }
-
- // Call closeExecutorService() via reflection
- java.lang.reflect.Method closeExecutorMethod =
- SnapshotSplitAssigner.class.getDeclaredMethod("closeExecutorService");
- closeExecutorMethod.setAccessible(true);
- closeExecutorMethod.invoke(snapshotSplitAssigner);
-
- // Call chunkSplitter.close() via reflection
- java.lang.reflect.Field chunkSplitterField =
- SnapshotSplitAssigner.class.getDeclaredField("chunkSplitter");
- chunkSplitterField.setAccessible(true);
- Object chunkSplitter = chunkSplitterField.get(snapshotSplitAssigner);
-
- if (chunkSplitter != null) {
- java.lang.reflect.Method closeMethod = chunkSplitter.getClass().getMethod("close");
- closeMethod.invoke(chunkSplitter);
- LOG.info("Closed Source chunkSplitter JDBC connection");
- }
- } catch (Exception e) {
- LOG.warn("Failed to close chunkSplitter via reflection", e);
- }
- }
-
// Method removed - reader cleanup is now handled in finishSplitRecords()
protected abstract FetchTask<SourceSplitBase> createFetchTaskFromSplit(
@@ -945,7 +907,7 @@ public abstract class JdbcIncrementalSourceReader extends AbstractCdcSourceReade
}
}
- private Map<TableId, TableChanges.TableChange> getTableSchemas(JobBaseConfig config) {
+ protected Map<TableId, TableChanges.TableChange> getTableSchemas(JobBaseConfig config) {
Map<TableId, TableChanges.TableChange> schemas = this.getTableSchemas();
if (schemas == null) {
schemas = discoverTableSchemas(config);
diff --git a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReader.java b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReader.java
index 84340f5e137..f576485c2f8 100644
--- a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReader.java
+++ b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReader.java
@@ -38,14 +38,15 @@ import org.apache.doris.job.cdc.split.SnapshotSplit;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.flink.api.connector.source.SourceSplit;
-import org.apache.flink.api.connector.source.mocks.MockSplitEnumeratorContext;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.cdc.common.utils.Preconditions;
import org.apache.flink.cdc.connectors.mysql.debezium.DebeziumUtils;
import org.apache.flink.cdc.connectors.mysql.debezium.reader.BinlogSplitReader;
import org.apache.flink.cdc.connectors.mysql.debezium.reader.SnapshotSplitReader;
import org.apache.flink.cdc.connectors.mysql.debezium.task.context.StatefulTaskContext;
-import org.apache.flink.cdc.connectors.mysql.source.assigners.MySqlSnapshotSplitAssigner;
+import org.apache.flink.cdc.connectors.mysql.schema.MySqlSchema;
+import org.apache.flink.cdc.connectors.mysql.source.assigners.MySqlChunkSplitter;
+import org.apache.flink.cdc.connectors.mysql.source.assigners.state.ChunkSplitterState;
import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfig;
import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfigFactory;
import org.apache.flink.cdc.connectors.mysql.source.offset.BinlogOffset;
@@ -83,7 +84,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
-import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
@@ -161,46 +161,126 @@ public class MySqlSourceReader extends AbstractCdcSourceReader {
LOG.info("Initialized poll executor with parallelism: {}", parallelism);
}
+ /**
+ * Fetch a batch of snapshot splits by driving flink-cdc {@link MySqlChunkSplitter} directly.
+ *
+ * <p>Stateless: each RPC rebuilds the splitter from (table, nextSplitStart, nextSplitId)
+ * supplied by FE, splits up to {@code batchSize} chunks, then closes. Note: evenly-distributed
+ * PKs go through a single splitChunks() call returning all chunks at once, so batchSize is only
+ * effective on the uneven path.
+ *
+ * <p>Only INITIAL/SNAPSHOT startup modes go through the chunk path; other modes return a single
+ * BinlogSplit instead.
+ */
@Override
public List<AbstractSourceSplit> getSourceSplits(FetchTableSplitsRequest ftsReq) {
- LOG.info("Get table {} splits for job {}", ftsReq.getSnapshotTable(), ftsReq.getJobId());
+ LOG.info(
+ "Get table {} splits for job {} (nextSplitId={}, nextSplitStart={})",
+ ftsReq.getSnapshotTable(),
+ ftsReq.getJobId(),
+ ftsReq.getNextSplitId(),
+ java.util.Arrays.toString(ftsReq.getNextSplitStart()));
MySqlSourceConfig sourceConfig = getSourceConfig(ftsReq);
StartupMode startupMode = sourceConfig.getStartupOptions().startupMode;
- List<MySqlSnapshotSplit> remainingSnapshotSplits = new ArrayList<>();
- MySqlBinlogSplit remainingBinlogSplit = null;
- if (startupMode.equals(StartupMode.INITIAL) || startupMode.equals(StartupMode.SNAPSHOT)) {
- remainingSnapshotSplits =
- startSplitChunks(sourceConfig, ftsReq.getSnapshotTable(), ftsReq.getConfig());
- } else {
- remainingBinlogSplit =
- new MySqlBinlogSplit(
- BINLOG_SPLIT_ID,
- sourceConfig.getStartupOptions().binlogOffset,
- BinlogOffset.ofNonStopping(),
- new ArrayList<>(),
- new HashMap<>(),
- 0);
+
+ if (!startupMode.equals(StartupMode.INITIAL) && !startupMode.equals(StartupMode.SNAPSHOT)) {
+ BinlogSplit binlogSplit = new BinlogSplit();
+ binlogSplit.setSplitId(BINLOG_SPLIT_ID);
+ binlogSplit.setStartingOffset(
+ sourceConfig.getStartupOptions().binlogOffset.getOffset());
+ return Collections.singletonList(binlogSplit);
}
- List<AbstractSourceSplit> splits = new ArrayList<>();
- if (!remainingSnapshotSplits.isEmpty()) {
- for (MySqlSnapshotSplit snapshotSplit : remainingSnapshotSplits) {
- String splitId = snapshotSplit.splitId();
- String tableId = snapshotSplit.getTableId().identifier();
- Object[] splitStart = snapshotSplit.getSplitStart();
- Object[] splitEnd = snapshotSplit.getSplitEnd();
- List<String> splitKey = snapshotSplit.getSplitKeyType().getFieldNames();
- SnapshotSplit split =
- new SnapshotSplit(splitId, tableId, splitKey, splitStart, splitEnd, null);
- splits.add(split);
+
+ String database = ftsReq.getConfig().get(DataSourceConfigKeys.DATABASE);
+ TableId tableId = TableId.parse(database + "." + ftsReq.getSnapshotTable());
+ int batchSize = ftsReq.getBatchSize() == null ? 100 : ftsReq.getBatchSize();
+
+ boolean isCaseSensitive;
+ try (MySqlConnection jdbc = DebeziumUtils.createMySqlConnection(sourceConfig)) {
+ isCaseSensitive = DebeziumUtils.isTableIdCaseSensitive(jdbc);
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to query table id case sensitivity", e);
+ }
+ MySqlSchema mySqlSchema = new MySqlSchema(sourceConfig, isCaseSensitive);
+ MySqlPartition partition =
+ new MySqlPartition(sourceConfig.getMySqlConnectorConfig().getLogicalName());
+
+ ChunkSplitterState state =
+ buildChunkSplitterState(sourceConfig, tableId, ftsReq, mySqlSchema, partition);
+ MySqlChunkSplitter splitter = new MySqlChunkSplitter(mySqlSchema, sourceConfig, state);
+
+ try {
+ splitter.open();
+ List<AbstractSourceSplit> result = new ArrayList<>();
+ while (result.size() < batchSize) {
+ List<MySqlSnapshotSplit> chunks = splitter.splitChunks(partition, tableId);
+ for (MySqlSnapshotSplit chunk : chunks) {
+ result.add(toDorisSnapshotSplit(chunk));
+ }
+ if (!splitter.hasNextChunk()) {
+ break;
+ }
+ }
+ LOG.info(
+ "Fetched {} splits for table {} (resume nextSplitId={}); hasNextChunk={}",
+ result.size(),
+ tableId,
+ ftsReq.getNextSplitId(),
+ splitter.hasNextChunk());
+ return result;
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to generate splits for " + tableId, e);
+ } finally {
+ try {
+ splitter.close();
+ } catch (Exception e) {
+ LOG.warn("Failed to close splitter for {}", tableId, e);
}
- } else {
- BinlogOffset startingOffset = remainingBinlogSplit.getStartingOffset();
- BinlogSplit binlogSplit = new BinlogSplit();
- binlogSplit.setSplitId(remainingBinlogSplit.splitId());
- binlogSplit.setStartingOffset(startingOffset.getOffset());
- splits.add(binlogSplit);
}
- return splits;
+ }
+
+ /**
+ * null start -> NO_SPLITTING_TABLE_STATE (analyze + maybe evenly); non-null -> resume
+ * mid-table. Cast pkValues[0] back to the JDBC driver's natural type (JSON round-trip
+ * downgrades types).
+ */
+ private ChunkSplitterState buildChunkSplitterState(
+ MySqlSourceConfig sourceConfig,
+ TableId tableId,
+ FetchTableSplitsRequest ftsReq,
+ MySqlSchema mySqlSchema,
+ MySqlPartition partition) {
+ Object[] pkValues = ftsReq.getNextSplitStart();
+ if (pkValues == null || pkValues.length == 0) {
+ return ChunkSplitterState.NO_SPLITTING_TABLE_STATE;
+ }
+ Column splitColumn;
+ try (MySqlConnection jdbc = DebeziumUtils.createMySqlConnection(sourceConfig)) {
+ splitColumn =
+ ChunkUtils.getChunkKeyColumn(
+ mySqlSchema.getTableSchema(partition, jdbc, tableId).getTable(),
+ sourceConfig.getChunkKeyColumns());
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to resolve split column for " + tableId, e);
+ }
+ Class<?> targetClass = resolveSplitKeyClass(tableId, splitColumn, ftsReq);
+ Object[] castStart = convertBounds(pkValues, targetClass, objectMapper);
+ int splitId = ftsReq.getNextSplitId() == null ? 0 : ftsReq.getNextSplitId();
+ return new ChunkSplitterState(
+ tableId, ChunkSplitterState.ChunkBound.middleOf(castStart[0]), splitId);
+ }
+
+ /**
+ * flink-cdc MySqlSnapshotSplit -> Doris SnapshotSplit (drops splitKeyType, keeps field names).
+ */
+ private SnapshotSplit toDorisSnapshotSplit(MySqlSnapshotSplit chunk) {
+ return new SnapshotSplit(
+ chunk.splitId(),
+ chunk.getTableId().identifier(),
+ chunk.getSplitKeyType().getFieldNames(),
+ chunk.getSplitStart(),
+ chunk.getSplitEnd(),
+ null);
}
@Override
@@ -731,66 +811,6 @@ public class MySqlSourceReader extends AbstractCdcSourceReader {
return Tuple2.of(binlogSplitFinal, pureBinlogPhase);
}
- private List<MySqlSnapshotSplit> startSplitChunks(
- MySqlSourceConfig sourceConfig, String snapshotTable, Map<String, String> config) {
- List<TableId> remainingTables = new ArrayList<>();
- if (snapshotTable != null) {
- // need add database name
- String database = config.get(DataSourceConfigKeys.DATABASE);
- remainingTables.add(TableId.parse(database + "." + snapshotTable));
- }
- List<MySqlSnapshotSplit> remainingSplits = new ArrayList<>();
- MySqlSnapshotSplitAssigner splitAssigner =
- new MySqlSnapshotSplitAssigner(
- sourceConfig, 1, remainingTables, false, new MockSplitEnumeratorContext(1));
- splitAssigner.open();
- try {
- while (true) {
- Optional<MySqlSplit> mySqlSplit = splitAssigner.getNext();
- if (mySqlSplit.isPresent()) {
- MySqlSnapshotSplit snapshotSplit = mySqlSplit.get().asSnapshotSplit();
- remainingSplits.add(snapshotSplit);
- } else {
- break;
- }
- }
- } finally {
- // splitAssigner.close();
- closeChunkSplitterOnly(splitAssigner);
- }
- return remainingSplits;
- }
-
- /**
- * The JdbcConnectionPools inside MySqlSnapshotSplitAssigner are singletons. Calling
- * MySqlSnapshotSplitAssigner.close() closes the entire JdbcConnectionPools, which can cause
- * problems under high concurrency. This only closes the connection of the current
- * MySqlSnapshotSplitAssigner.
- */
- private void closeChunkSplitterOnly(MySqlSnapshotSplitAssigner splitAssigner) {
- try {
- // call closeExecutorService()
- java.lang.reflect.Method closeExecutorMethod =
- MySqlSnapshotSplitAssigner.class.getDeclaredMethod("closeExecutorService");
- closeExecutorMethod.setAccessible(true);
- closeExecutorMethod.invoke(splitAssigner);
-
- // call chunkSplitter.close()
- java.lang.reflect.Field field =
- MySqlSnapshotSplitAssigner.class.getDeclaredField("chunkSplitter");
- field.setAccessible(true);
- Object chunkSplitter = field.get(splitAssigner);
-
- if (chunkSplitter != null) {
- java.lang.reflect.Method closeMethod = chunkSplitter.getClass().getMethod("close");
- closeMethod.invoke(chunkSplitter);
- LOG.info("Closed chunkSplitter JDBC connection");
- }
- } catch (Exception e) {
- LOG.warn("Failed to close chunkSplitter via reflection,", e);
- }
- }
-
private SnapshotSplitReader getSnapshotSplitReader(JobBaseConfig config, int subtaskId) {
MySqlSourceConfig sourceConfig = getSourceConfig(config);
final MySqlConnection jdbcConnection = DebeziumUtils.createMySqlConnection(sourceConfig);
diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_async_split.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_async_split.groovy
new file mode 100644
index 00000000000..8275bd70c72
--- /dev/null
+++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_async_split.groovy
@@ -0,0 +1,160 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+import org.awaitility.Awaitility
+
+import static java.util.concurrent.TimeUnit.SECONDS
+
+// Verifies async chunk splitting end-to-end for MySQL: CREATE returns quickly,
+// snapshot fully syncs across multiple splits, and the binlog phase still works after.
+suite("test_streaming_mysql_job_async_split", "p0,external,mysql,external_docker,external_docker_mysql,nondatalake") {
+ def jobName = "test_streaming_mysql_job_async_split_name"
+ def currentDb = (sql "select database()")[0][0]
+ def table1 = "user_info_mysql_async_split"
+ def mysqlDb = "test_cdc_db"
+ def totalRows = 100
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ sql """drop table if exists ${currentDb}.${table1} force"""
+
+ String enabled = context.config.otherConfigs.get("enableJdbcTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String mysql_port = context.config.otherConfigs.get("mysql_57_port");
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String s3_endpoint = getS3Endpoint()
+ String bucket = getS3BucketName()
+ String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.4.0.jar"
+
+ // Prepare 100 rows in MySQL; snapshot_split_size=5 -> 20 splits.
+ connect("root", "123456", "jdbc:mysql://${externalEnvIp}:${mysql_port}") {
+ sql """CREATE DATABASE IF NOT EXISTS ${mysqlDb}"""
+ sql """DROP TABLE IF EXISTS ${mysqlDb}.${table1}"""
+ sql """CREATE TABLE ${mysqlDb}.${table1} (
+ `id` int NOT NULL,
+ `name` varchar(200),
+ `age` int,
+ PRIMARY KEY (`id`)
+ ) ENGINE=InnoDB"""
+ StringBuilder sb = new StringBuilder()
+ sb.append("INSERT INTO ${mysqlDb}.${table1} (id, name, age) VALUES ")
+ for (int i = 1; i <= totalRows; i++) {
+ if (i > 1) sb.append(", ")
+ sb.append("(${i}, 'name_${i}', ${i})")
+ }
+ sql sb.toString()
+ }
+
+ // CREATE should return promptly: splitting is now driven by the scheduler
+ // each tick, not synchronously inside CREATE.
+ long createStartMs = System.currentTimeMillis()
+ sql """CREATE JOB ${jobName}
+ ON STREAMING
+ FROM MYSQL (
+ "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "com.mysql.cj.jdbc.Driver",
+ "user" = "root",
+ "password" = "123456",
+ "database" = "${mysqlDb}",
+ "include_tables" = "${table1}",
+ "offset" = "initial",
+ "snapshot_split_size" = "5",
+ "snapshot_parallelism" = "4"
+ )
+ TO DATABASE ${currentDb} (
+ "table.create.properties.replication_num" = "1"
+ )
+ """
+ long createElapsedMs = System.currentTimeMillis() - createStartMs
+ log.info("CREATE JOB elapsed: ${createElapsedMs} ms")
+
+ // Job metadata should be visible immediately (no blocking on splitChunks).
+ def jobRows = sql """select Name from jobs("type"="insert") where Name='${jobName}'"""
+ assert jobRows.size() == 1
+
+ // Wait until snapshot fully syncs. 20 splits at snapshot_parallelism=4 ->
+ // ~5 task ticks; default max_interval=10s -> ~50s end-to-end. Cap at 5min.
+ try {
+ Awaitility.await().atMost(300, SECONDS)
+ .pollInterval(2, SECONDS).until(
+ {
+ def cnt = sql """SELECT COUNT(*) FROM ${currentDb}.${table1}"""
+ log.info("doris row count: ${cnt}")
+ cnt.size() == 1 && cnt.get(0).get(0) == totalRows
+ }
+ )
+ } catch (Exception ex) {
+ def showjob = sql """select * from jobs("type"="insert") where Name='${jobName}'"""
+ def showtask = sql """select * from tasks("type"="insert") where JobName='${jobName}'"""
+ log.info("show job: " + showjob)
+ log.info("show task: " + showtask)
+ throw ex
+ }
+
+ // Distinct PK count must equal total — no duplicates from RPC retry / dedup paths.
+ def doneDistinct = sql """SELECT COUNT(DISTINCT id) FROM ${currentDb}.${table1}"""
+ assert doneDistinct.get(0).get(0) == totalRows
+
+ def loadStat0 = parseJson(sql("""
+ select loadStatistic from jobs("type"="insert") where Name='${jobName}'
+ """).get(0).get(0))
+ log.info("loadStat after snapshot: ${loadStat0}")
+ assert loadStat0.scannedRows == totalRows
+
+ // Snapshot done -> binlog phase. Verify INSERT/UPDATE/DELETE still propagate.
+ connect("root", "123456", "jdbc:mysql://${externalEnvIp}:${mysql_port}") {
+ sql """INSERT INTO ${mysqlDb}.${table1} (id, name, age) VALUES (101, 'incr_101', 101);"""
+ sql """UPDATE ${mysqlDb}.${table1} SET age = 999 WHERE id = 1;"""
+ sql """DELETE FROM ${mysqlDb}.${table1} WHERE id = 2;"""
+ }
+
+ try {
+ Awaitility.await().atMost(120, SECONDS)
+ .pollInterval(2, SECONDS).until(
+ {
+ def res = sql """SELECT
+ COUNT(*),
+ SUM(CASE WHEN id=1 AND age=999 THEN 1 ELSE 0 END),
+ SUM(CASE WHEN id=101 THEN 1 ELSE 0 END),
+ SUM(CASE WHEN id=2 THEN 1 ELSE 0 END)
+ FROM ${currentDb}.${table1}"""
+ log.info("binlog state: ${res}")
+ res.size() == 1
+ && res.get(0).get(0) == totalRows
+ && res.get(0).get(1) == 1
+ && res.get(0).get(2) == 1
+ && res.get(0).get(3) == 0
+ }
+ )
+ } catch (Exception ex) {
+ def showjob = sql """select * from jobs("type"="insert") where Name='${jobName}'"""
+ log.info("show job: " + showjob)
+ throw ex
+ }
+
+ // Job should still be RUNNING (snapshot transitioned cleanly to binlog).
+ def jobInfo = sql """
+ select status from jobs("type"="insert") where Name='${jobName}'
+ """
+ assert jobInfo.get(0).get(0) == "RUNNING"
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ def jobCountRsp = sql """select count(1) from jobs("type"="insert") where Name ='${jobName}'"""
+ assert jobCountRsp.get(0).get(0) == 0
+ }
+}
diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_async_split_uneven.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_async_split_uneven.groovy
new file mode 100644
index 00000000000..95d5fc16aa5
--- /dev/null
+++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_async_split_uneven.groovy
@@ -0,0 +1,195 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+import org.awaitility.Awaitility
+
+import java.util.concurrent.CopyOnWriteArraySet
+import java.util.concurrent.atomic.AtomicBoolean
+
+import static java.util.concurrent.TimeUnit.SECONDS
+
+// MySQL uneven splitter path — VARCHAR PK forces splitOneUnevenlySizedChunk on the
+// MySQL side (separate impl from PG: MySqlSourceReader.resolveSplitKeyClass lives
+// in cdc_client). 1500 rows / split_size=5 = 300 chunks > default batch_size=100,
+// so chunk_list growth is observably stepped across ~3 RPCs without mutating the
+// global FE config (which would race with concurrent cdc cases).
+suite("test_streaming_mysql_job_async_split_uneven",
+ "p0,external,mysql,external_docker,external_docker_mysql,nondatalake") {
+ def jobName = "test_streaming_mysql_job_async_split_uneven_name"
+ def currentDb = (sql "select database()")[0][0]
+ def table1 = "user_info_mysql_async_split_uneven"
+ def mysqlDb = "test_cdc_db"
+ def totalRows = 1500
+ int expectedChunks = (int) Math.ceil(totalRows / 5.0) // 300 chunks at split_size=5
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ sql """drop table if exists ${currentDb}.${table1} force"""
+
+ String enabled = context.config.otherConfigs.get("enableJdbcTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String mysql_port = context.config.otherConfigs.get("mysql_57_port");
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String s3_endpoint = getS3Endpoint()
+ String bucket = getS3BucketName()
+ String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.4.0.jar"
+
+ connect("root", "123456", "jdbc:mysql://${externalEnvIp}:${mysql_port}") {
+ sql """CREATE DATABASE IF NOT EXISTS ${mysqlDb}"""
+ sql """DROP TABLE IF EXISTS ${mysqlDb}.${table1}"""
+ sql """CREATE TABLE ${mysqlDb}.${table1} (
+ `id` varchar(20) NOT NULL,
+ `name` varchar(200),
+ `age` int,
+ PRIMARY KEY (`id`)
+ ) ENGINE=InnoDB"""
+ StringBuilder sb = new StringBuilder()
+ sb.append("INSERT INTO ${mysqlDb}.${table1} (id, name, age) VALUES ")
+ for (int i = 1; i <= totalRows; i++) {
+ if (i > 1) sb.append(", ")
+ String key = "k_" + String.format("%05d", i)
+ sb.append("('${key}', 'name_${i}', ${i})")
+ }
+ sql sb.toString()
+ }
+
+ sql """CREATE JOB ${jobName}
+ ON STREAMING
+ FROM MYSQL (
+ "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "com.mysql.cj.jdbc.Driver",
+ "user" = "root",
+ "password" = "123456",
+ "database" = "${mysqlDb}",
+ "include_tables" = "${table1}",
+ "offset" = "initial",
+ "snapshot_split_size" = "5",
+ "snapshot_parallelism" = "2"
+ )
+ TO DATABASE ${currentDb} (
+ "table.create.properties.replication_num" = "1"
+ )
+ """
+
+ def jobIdRow = sql """select Id from jobs("type"="insert") where Name='${jobName}'"""
+ assert jobIdRow.size() == 1
+ def jobId = jobIdRow.get(0).get(0).toString()
+
+ // Background sampler: poll streaming_job_meta.chunk_list length every 150ms while
+ // the splitter runs. With default batch_size=100 and 300 chunks the splitter does
+ // ~3 RPCs so we should observe >=3 distinct lengths (100, 200, 300).
+ def lengthsSeen = new CopyOnWriteArraySet<Integer>()
+ def samplerStop = new AtomicBoolean(false)
+ def sampler = Thread.start {
+ int lastSeen = -1
+ while (!samplerStop.get()) {
+ try {
+ def r = sql """SELECT json_length(chunk_list)
+ FROM internal.__internal_schema.streaming_job_meta
+ WHERE job_id='${jobId}'"""
+ if (r.size() > 0 && r.get(0).get(0) != null) {
+ int len = r.get(0).get(0) as int
+ lengthsSeen.add(len)
+ if (len != lastSeen) {
+ log.info("sampler: chunk_list length=${len}")
+ lastSeen = len
+ }
+ // Assertion only needs >=3 distinct lengths; stop polling once satisfied
+ // to keep the case log slim during the remaining snapshot consumption.
+ if (lengthsSeen.size() >= 3) break
+ }
+ } catch (Throwable ignored) { /* meta row not visible yet */ }
+ try { Thread.sleep(150) } catch (InterruptedException ie) { break }
+ }
+ }
+
+ try {
+ Awaitility.await().atMost(600, SECONDS)
+ .pollInterval(2, SECONDS).until(
+ {
+ def cnt = sql """SELECT COUNT(*) FROM ${currentDb}.${table1}"""
+ log.info("doris row count: ${cnt}")
+ cnt.size() == 1 && cnt.get(0).get(0) == totalRows
+ }
+ )
+ } catch (Exception ex) {
+ samplerStop.set(true)
+ def showjob = sql """select * from jobs("type"="insert") where Name='${jobName}'"""
+ def showtask = sql """select * from tasks("type"="insert") where JobName='${jobName}'"""
+ log.info("show job: " + showjob)
+ log.info("show task: " + showtask)
+ throw ex
+ }
+ samplerStop.set(true)
+ sampler.join(5000)
+ log.info("chunk_list lengths observed during snapshot: ${lengthsSeen.toSorted()}")
+
+ // >=3 distinct lengths = chunk_list grew in batched UPSERTs; one-shot would show only one.
+ assert lengthsSeen.size() >= 3 :
+ "MySQL uneven splitting should produce chunk_list in batches (>=3 distinct lengths)," +
+ " saw only ${lengthsSeen.toSorted()} — implies one-shot splitting"
+
+ def doneDistinct = sql """SELECT COUNT(DISTINCT id) FROM ${currentDb}.${table1}"""
+ assert doneDistinct.get(0).get(0) == totalRows :
+ "DISTINCT pk count ${doneDistinct.get(0).get(0)} != ${totalRows} — chunks may have re-cut"
+
+ def loadStat0 = parseJson(sql("""
+ select loadStatistic from jobs("type"="insert") where Name='${jobName}'
+ """).get(0).get(0))
+ log.info("loadStat after MySQL uneven snapshot: ${loadStat0}")
+ assert loadStat0.scannedRows == totalRows
+
+ // Binlog phase: INSERT/UPDATE/DELETE must all propagate after the uneven snapshot.
+ connect("root", "123456", "jdbc:mysql://${externalEnvIp}:${mysql_port}") {
+ sql """INSERT INTO ${mysqlDb}.${table1} (id, name, age) VALUES ('k_99999', 'incr', 999);"""
+ sql """UPDATE ${mysqlDb}.${table1} SET age = 8888 WHERE id = 'k_00001';"""
+ sql """DELETE FROM ${mysqlDb}.${table1} WHERE id = 'k_00002';"""
+ }
+
+ try {
+ Awaitility.await().atMost(120, SECONDS)
+ .pollInterval(2, SECONDS).until(
+ {
+ def res = sql """SELECT
+ COUNT(*),
+ SUM(CASE WHEN id='k_00001' AND age=8888 THEN 1 ELSE 0 END),
+ SUM(CASE WHEN id='k_99999' THEN 1 ELSE 0 END),
+ SUM(CASE WHEN id='k_00002' THEN 1 ELSE 0 END)
+ FROM ${currentDb}.${table1}"""
+ log.info("binlog state: ${res}")
+ res.size() == 1
+ && res.get(0).get(0) == totalRows
+ && res.get(0).get(1) == 1
+ && res.get(0).get(2) == 1
+ && res.get(0).get(3) == 0
+ }
+ )
+ } catch (Exception ex) {
+ def showjob = sql """select * from jobs("type"="insert") where Name='${jobName}'"""
+ log.info("show job: " + showjob)
+ throw ex
+ }
+
+ def jobInfo = sql """select status from jobs("type"="insert") where Name='${jobName}'"""
+ assert jobInfo.get(0).get(0) == "RUNNING"
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ def jobCountRsp = sql """select count(1) from jobs("type"="insert") where Name='${jobName}'"""
+ assert jobCountRsp.get(0).get(0) == 0
+ }
+}
diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_async_split_multi_table.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_async_split_multi_table.groovy
new file mode 100644
index 00000000000..0bdcc80aea4
--- /dev/null
+++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_async_split_multi_table.groovy
@@ -0,0 +1,178 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+import org.awaitility.Awaitility
+
+import static java.util.concurrent.TimeUnit.SECONDS
+
+// Verifies cross-table progression of currentSplittingTable: 3 tables, mixed PK types
+// (int4 -> evenly path, varchar -> uneven path), each must finish fully before the
+// next starts splitting; binlog phase then drains DML for every table.
+suite("test_streaming_postgres_job_async_split_multi_table",
+ "p0,external,pg,external_docker,external_docker_pg,nondatalake") {
+ def jobName = "test_streaming_postgres_job_async_split_multi_table_name"
+ def currentDb = (sql "select database()")[0][0]
+ def t1 = "user_info_pg_async_split_mt_a" // int4 PK, evenly path
+ def t2 = "user_info_pg_async_split_mt_b" // varchar PK, uneven path
+ def t3 = "user_info_pg_async_split_mt_c" // int4 PK, evenly path
+ def pgDB = "postgres"
+ def pgSchema = "cdc_test"
+ def pgUser = "postgres"
+ def pgPassword = "123456"
+ def rowsA = 300
+ def rowsB = 200
+ def rowsC = 250
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ sql """drop table if exists ${currentDb}.${t1} force"""
+ sql """drop table if exists ${currentDb}.${t2} force"""
+ sql """drop table if exists ${currentDb}.${t3} force"""
+
+ String enabled = context.config.otherConfigs.get("enableJdbcTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String pg_port = context.config.otherConfigs.get("pg_14_port");
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String s3_endpoint = getS3Endpoint()
+ String bucket = getS3BucketName()
+ String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/postgresql-42.5.0.jar"
+
+ connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") {
+ sql """DROP TABLE IF EXISTS ${pgDB}.${pgSchema}.${t1}"""
+ sql """DROP TABLE IF EXISTS ${pgDB}.${pgSchema}.${t2}"""
+ sql """DROP TABLE IF EXISTS ${pgDB}.${pgSchema}.${t3}"""
+
+ sql """CREATE TABLE ${pgDB}.${pgSchema}.${t1} (
+ "id" int4 PRIMARY KEY,
+ "name" varchar(200)
+ )"""
+ StringBuilder sb1 = new StringBuilder("INSERT INTO ${pgDB}.${pgSchema}.${t1} (id, name) VALUES ")
+ for (int i = 1; i <= rowsA; i++) {
+ if (i > 1) sb1.append(", ")
+ sb1.append("(${i}, 'a_${i}')")
+ }
+ sql sb1.toString()
+
+ sql """CREATE TABLE ${pgDB}.${pgSchema}.${t2} (
+ "id" varchar(20) PRIMARY KEY,
+ "name" varchar(200)
+ )"""
+ StringBuilder sb2 = new StringBuilder("INSERT INTO ${pgDB}.${pgSchema}.${t2} (id, name) VALUES ")
+ for (int i = 1; i <= rowsB; i++) {
+ if (i > 1) sb2.append(", ")
+ String key = "k_" + String.format("%05d", i)
+ sb2.append("('${key}', 'b_${i}')")
+ }
+ sql sb2.toString()
+
+ sql """CREATE TABLE ${pgDB}.${pgSchema}.${t3} (
+ "id" int4 PRIMARY KEY,
+ "name" varchar(200)
+ )"""
+ StringBuilder sb3 = new StringBuilder("INSERT INTO ${pgDB}.${pgSchema}.${t3} (id, name) VALUES ")
+ for (int i = 1; i <= rowsC; i++) {
+ if (i > 1) sb3.append(", ")
+ sb3.append("(${i}, 'c_${i}')")
+ }
+ sql sb3.toString()
+ }
+
+ long createStartMs = System.currentTimeMillis()
+ sql """CREATE JOB ${jobName}
+ ON STREAMING
+ FROM POSTGRES (
+ "jdbc_url" = "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "org.postgresql.Driver",
+ "user" = "${pgUser}",
+ "password" = "${pgPassword}",
+ "database" = "${pgDB}",
+ "schema" = "${pgSchema}",
+ "include_tables" = "${t1},${t2},${t3}",
+ "offset" = "initial",
+ "snapshot_split_size" = "10",
+ "snapshot_parallelism" = "4"
+ )
+ TO DATABASE ${currentDb} (
+ "table.create.properties.replication_num" = "1"
+ )
+ """
+ long createElapsedMs = System.currentTimeMillis() - createStartMs
+ log.info("CREATE JOB elapsed (3 tables): ${createElapsedMs} ms")
+
+ def jobRows = sql """select Name from jobs("type"="insert") where Name='${jobName}'"""
+ assert jobRows.size() == 1
+
+ try {
+ Awaitility.await().atMost(600, SECONDS)
+ .pollInterval(2, SECONDS).until(
+ {
+ def a = sql """SELECT COUNT(*) FROM ${currentDb}.${t1}"""
+ def b = sql """SELECT COUNT(*) FROM ${currentDb}.${t2}"""
+ def c = sql """SELECT COUNT(*) FROM ${currentDb}.${t3}"""
+ log.info("a=${a.get(0).get(0)} b=${b.get(0).get(0)} c=${c.get(0).get(0)}")
+ a.get(0).get(0) == rowsA && b.get(0).get(0) == rowsB && c.get(0).get(0) == rowsC
+ }
+ )
+ } catch (Exception ex) {
+ def showjob = sql """select * from jobs("type"="insert") where Name='${jobName}'"""
+ def showtask = sql """select * from tasks("type"="insert") where JobName='${jobName}'"""
+ log.info("show job: " + showjob)
+ log.info("show task: " + showtask)
+ throw ex
+ }
+
+ // DISTINCT per table guards against cross-table chunk leakage / dedup misses.
+ assert sql("""SELECT COUNT(DISTINCT id) FROM ${currentDb}.${t1}""").get(0).get(0) == rowsA
+ assert sql("""SELECT COUNT(DISTINCT id) FROM ${currentDb}.${t2}""").get(0).get(0) == rowsB
+ assert sql("""SELECT COUNT(DISTINCT id) FROM ${currentDb}.${t3}""").get(0).get(0) == rowsC
+
+ def loadStat = parseJson(sql("""
+ select loadStatistic from jobs("type"="insert") where Name='${jobName}'
+ """).get(0).get(0))
+ log.info("loadStat after multi-table snapshot: ${loadStat}")
+ assert loadStat.scannedRows == (rowsA + rowsB + rowsC)
+
+ // Binlog phase on every table — currentOffset has to drive all 3.
+ connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") {
+ sql """INSERT INTO ${pgDB}.${pgSchema}.${t1} (id, name) VALUES (9991, 'a_incr');"""
+ sql """INSERT INTO ${pgDB}.${pgSchema}.${t2} (id, name) VALUES ('k_99991', 'b_incr');"""
+ sql """INSERT INTO ${pgDB}.${pgSchema}.${t3} (id, name) VALUES (9993, 'c_incr');"""
+ }
+
+ try {
+ Awaitility.await().atMost(120, SECONDS)
+ .pollInterval(2, SECONDS).until(
+ {
+ def a = sql """SELECT count(*) FROM ${currentDb}.${t1} WHERE id=9991"""
+ def b = sql """SELECT count(*) FROM ${currentDb}.${t2} WHERE id='k_99991'"""
+ def c = sql """SELECT count(*) FROM ${currentDb}.${t3} WHERE id=9993"""
+ a.get(0).get(0) == 1 && b.get(0).get(0) == 1 && c.get(0).get(0) == 1
+ }
+ )
+ } catch (Exception ex) {
+ def showjob = sql """select * from jobs("type"="insert") where Name='${jobName}'"""
+ log.info("show job: " + showjob)
+ throw ex
+ }
+
+ def status = sql """select status from jobs("type"="insert") where Name='${jobName}'"""
+ assert status.get(0).get(0) == "RUNNING"
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ }
+}
diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_async_split_pause_resume.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_async_split_pause_resume.groovy
new file mode 100644
index 00000000000..c367f4646e2
--- /dev/null
+++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_async_split_pause_resume.groovy
@@ -0,0 +1,172 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+import org.awaitility.Awaitility
+
+import static java.util.concurrent.TimeUnit.SECONDS
+
+// PAUSE mid-snapshot must stopAsyncWork the splitter thread (SucceedTaskCount stops
+// growing); RESUME must startAsyncWorkIfNeed and continue from cdcSplitProgress
+// without re-cutting committed chunks. Uses uneven path so the snapshot is slow
+// enough to catch mid-flight.
+suite("test_streaming_postgres_job_async_split_pause_resume",
+ "p0,external,pg,external_docker,external_docker_pg,nondatalake") {
+ def jobName = "test_streaming_postgres_job_async_split_pause_resume_name"
+ def currentDb = (sql "select database()")[0][0]
+ def table1 = "user_info_pg_async_split_pause_resume"
+ def pgDB = "postgres"
+ def pgSchema = "cdc_test"
+ def pgUser = "postgres"
+ def pgPassword = "123456"
+ def totalRows = 200
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ sql """drop table if exists ${currentDb}.${table1} force"""
+
+ String enabled = context.config.otherConfigs.get("enableJdbcTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String pg_port = context.config.otherConfigs.get("pg_14_port");
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String s3_endpoint = getS3Endpoint()
+ String bucket = getS3BucketName()
+ String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/postgresql-42.5.0.jar"
+
+ connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") {
+ sql """DROP TABLE IF EXISTS ${pgDB}.${pgSchema}.${table1}"""
+ sql """CREATE TABLE ${pgDB}.${pgSchema}.${table1} (
+ "id" varchar(20) PRIMARY KEY,
+ "name" varchar(200)
+ )"""
+ StringBuilder sb = new StringBuilder()
+ sb.append("INSERT INTO ${pgDB}.${pgSchema}.${table1} (id, name) VALUES ")
+ for (int i = 1; i <= totalRows; i++) {
+ if (i > 1) sb.append(", ")
+ String key = "k_" + String.format("%05d", i)
+ sb.append("('${key}', 'name_${i}')")
+ }
+ sql sb.toString()
+ }
+
+ sql """CREATE JOB ${jobName}
+ ON STREAMING
+ FROM POSTGRES (
+ "jdbc_url" = "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "org.postgresql.Driver",
+ "user" = "${pgUser}",
+ "password" = "${pgPassword}",
+ "database" = "${pgDB}",
+ "schema" = "${pgSchema}",
+ "include_tables" = "${table1}",
+ "offset" = "initial",
+ "snapshot_split_size" = "5",
+ "snapshot_parallelism" = "1"
+ )
+ TO DATABASE ${currentDb} (
+ "table.create.properties.replication_num" = "1"
+ )
+ """
+
+ // Catch the job mid-snapshot — wait for some progress, but not finished.
+ try {
+ Awaitility.await().atMost(180, SECONDS)
+ .pollInterval(1, SECONDS).until(
+ {
+ def succeed = sql """select SucceedTaskCount from jobs("type"="insert") where Name = '${jobName}'"""
+ def cnt = sql """SELECT COUNT(*) FROM ${currentDb}.${table1}"""
+ log.info("pre-pause succeed=${succeed} rows=${cnt}")
+ succeed.size() == 1 && Integer.parseInt(succeed.get(0).get(0).toString()) >= 2
+ && cnt.get(0).get(0) < totalRows
+ }
+ )
+ } catch (Exception ex) {
+ log.info("job: " + (sql """select * from jobs("type"="insert") where Name='${jobName}'"""))
+ throw ex
+ }
+
+ def rowsBeforePause = sql("""SELECT COUNT(*) FROM ${currentDb}.${table1}""").get(0).get(0) as int
+ assert rowsBeforePause < totalRows : "snapshot finished before we could pause"
+
+ // PAUSE — splitter thread should stop within a few ticks.
+ sql """PAUSE JOB where jobname = '${jobName}'"""
+ try {
+ Awaitility.await().atMost(30, SECONDS).pollInterval(1, SECONDS).until({
+ def status = sql """select status from jobs("type"="insert") where Name='${jobName}'"""
+ status.size() == 1 && status.get(0).get(0) == "PAUSED"
+ })
+ } catch (Exception ex) {
+ log.info("job: " + (sql """select * from jobs("type"="insert") where Name='${jobName}'"""))
+ throw ex
+ }
+
+ // Capture state, sleep, recapture — succeedTaskCount must not grow while paused.
+ def succeedAtPause = sql("""select SucceedTaskCount from jobs("type"="insert") where Name='${jobName}'""")
+ .get(0).get(0).toString()
+ def rowsAtPause = sql("""SELECT COUNT(*) FROM ${currentDb}.${table1}""").get(0).get(0) as int
+ sleep(15000)
+ def succeedAfterSleep = sql("""select SucceedTaskCount from jobs("type"="insert") where Name='${jobName}'""")
+ .get(0).get(0).toString()
+ def rowsAfterSleep = sql("""SELECT COUNT(*) FROM ${currentDb}.${table1}""").get(0).get(0) as int
+ log.info("paused: succeed ${succeedAtPause}->${succeedAfterSleep} rows ${rowsAtPause}->${rowsAfterSleep}")
+ assert succeedAfterSleep == succeedAtPause :
+ "SucceedTaskCount grew while paused (${succeedAtPause} -> ${succeedAfterSleep}) — splitter not stopped"
+ assert rowsAfterSleep == rowsAtPause :
+ "row count grew while paused (${rowsAtPause} -> ${rowsAfterSleep}) — tasks still running"
+
+ def pausedStatus = sql """select status from jobs("type"="insert") where Name='${jobName}'"""
+ assert pausedStatus.get(0).get(0) == "PAUSED" : "job didn't stay PAUSED"
+
+ // RESUME — splitter thread should restart from cdcSplitProgress.
+ sql """RESUME JOB where jobname = '${jobName}'"""
+ try {
+ Awaitility.await().atMost(60, SECONDS).pollInterval(1, SECONDS).until({
+ def status = sql """select status from jobs("type"="insert") where Name='${jobName}'"""
+ status.size() == 1 && status.get(0).get(0) == "RUNNING"
+ })
+ } catch (Exception ex) {
+ log.info("job: " + (sql """select * from jobs("type"="insert") where Name='${jobName}'"""))
+ throw ex
+ }
+
+ try {
+ Awaitility.await().atMost(600, SECONDS)
+ .pollInterval(2, SECONDS).until(
+ {
+ def cnt = sql """SELECT COUNT(*) FROM ${currentDb}.${table1}"""
+ log.info("post-resume rows: ${cnt}")
+ cnt.size() == 1 && cnt.get(0).get(0) == totalRows
+ }
+ )
+ } catch (Exception ex) {
+ log.info("job: " + (sql """select * from jobs("type"="insert") where Name='${jobName}'"""))
+ log.info("tasks: " + (sql """select * from tasks("type"="insert") where JobName='${jobName}'"""))
+ throw ex
+ }
+
+ // DISTINCT must equal totalRows — resume must not re-cut committed chunks.
+ def distinctCnt = sql """SELECT COUNT(DISTINCT id) FROM ${currentDb}.${table1}"""
+ assert distinctCnt.get(0).get(0) == totalRows :
+ "DISTINCT count ${distinctCnt.get(0).get(0)} != ${totalRows} — resume re-cut chunks"
+
+ def jobInfo = sql """select status, FailedTaskCount, ErrorMsg from jobs("type"="insert") where Name='${jobName}'"""
+ assert jobInfo.get(0).get(0) == "RUNNING"
+ assert (jobInfo.get(0).get(1) as int) == 0 : "FailedTaskCount should be 0 after resume"
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ }
+}
diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_async_split_uneven.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_async_split_uneven.groovy
new file mode 100644
index 00000000000..bc0c893fb01
--- /dev/null
+++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_async_split_uneven.groovy
@@ -0,0 +1,198 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+import org.awaitility.Awaitility
+
+import java.util.concurrent.CopyOnWriteArraySet
+import java.util.concurrent.atomic.AtomicBoolean
+
+import static java.util.concurrent.TimeUnit.SECONDS
+
+// Exercises the uneven splitter path (VARCHAR PK bypasses isEvenlySplitColumn() and
+// goes through splitOneUnevenlySizedChunk -> per-chunk MAX(...) probe SQL) AND verifies
+// that splitting actually happens in batches: with 1500 rows / split_size=5 = 300 chunks
+// > default batch_size=100, the splitter takes ~3 RPCs so streaming_job_meta.chunk_list
+// grows incrementally (>=3 distinct lengths observed during snapshot) — a single-shot
+// synchronous splitter would write the full list in one UPSERT.
+suite("test_streaming_postgres_job_async_split_uneven", "p0,external,pg,external_docker,external_docker_pg,nondatalake") {
+ def jobName = "test_streaming_postgres_job_async_split_uneven_name"
+ def currentDb = (sql "select database()")[0][0]
+ def table1 = "user_info_pg_async_split_uneven"
+ def pgDB = "postgres"
+ def pgSchema = "cdc_test"
+ def pgUser = "postgres"
+ def pgPassword = "123456"
+ def totalRows = 1500
+ int expectedChunks = (int) Math.ceil(totalRows / 5.0) // 300 chunks at split_size=5
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ sql """drop table if exists ${currentDb}.${table1} force"""
+
+ String enabled = context.config.otherConfigs.get("enableJdbcTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String pg_port = context.config.otherConfigs.get("pg_14_port");
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String s3_endpoint = getS3Endpoint()
+ String bucket = getS3BucketName()
+ String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/postgresql-42.5.0.jar"
+
+ connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") {
+ sql """DROP TABLE IF EXISTS ${pgDB}.${pgSchema}.${table1}"""
+ sql """CREATE TABLE ${pgDB}.${pgSchema}.${table1} (
+ "id" varchar(20) PRIMARY KEY,
+ "name" varchar(200),
+ "age" int4
+ )"""
+ StringBuilder sb = new StringBuilder()
+ sb.append("INSERT INTO ${pgDB}.${pgSchema}.${table1} (id, name, age) VALUES ")
+ for (int i = 1; i <= totalRows; i++) {
+ if (i > 1) sb.append(", ")
+ String key = "k_" + String.format("%05d", i)
+ sb.append("('${key}', 'name_${i}', ${i})")
+ }
+ sql sb.toString()
+ }
+
+ sql """CREATE JOB ${jobName}
+ ON STREAMING
+ FROM POSTGRES (
+ "jdbc_url" = "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "org.postgresql.Driver",
+ "user" = "${pgUser}",
+ "password" = "${pgPassword}",
+ "database" = "${pgDB}",
+ "schema" = "${pgSchema}",
+ "include_tables" = "${table1}",
+ "offset" = "initial",
+ "snapshot_split_size" = "5",
+ "snapshot_parallelism" = "2"
+ )
+ TO DATABASE ${currentDb} (
+ "table.create.properties.replication_num" = "1"
+ )
+ """
+
+ def jobIdRow = sql """select Id from jobs("type"="insert") where Name='${jobName}'"""
+ assert jobIdRow.size() == 1
+ def jobId = jobIdRow.get(0).get(0).toString()
+
+ // Background sampler: poll streaming_job_meta.chunk_list length every 150ms while
+ // the splitter runs. With default batch_size=100 and 300 chunks the splitter does
+ // ~3 RPCs so we should observe >=3 distinct lengths (100, 200, 300).
+ def lengthsSeen = new CopyOnWriteArraySet<Integer>()
+ def samplerStop = new AtomicBoolean(false)
+ def sampler = Thread.start {
+ int lastSeen = -1
+ while (!samplerStop.get()) {
+ try {
+ def r = sql """SELECT json_length(chunk_list)
+ FROM internal.__internal_schema.streaming_job_meta
+ WHERE job_id='${jobId}'"""
+ if (r.size() > 0 && r.get(0).get(0) != null) {
+ int len = r.get(0).get(0) as int
+ lengthsSeen.add(len)
+ if (len != lastSeen) {
+ log.info("sampler: chunk_list length=${len}")
+ lastSeen = len
+ }
+ // Assertion only needs >=3 distinct lengths; stop polling once satisfied
+ // to keep the case log slim during the remaining snapshot consumption.
+ if (lengthsSeen.size() >= 3) break
+ }
+ } catch (Throwable ignored) { /* table may not exist yet; retry */ }
+ try { Thread.sleep(150) } catch (InterruptedException ie) { break }
+ }
+ }
+
+ try {
+ Awaitility.await().atMost(600, SECONDS)
+ .pollInterval(2, SECONDS).until(
+ {
+ def cnt = sql """SELECT COUNT(*) FROM ${currentDb}.${table1}"""
+ log.info("doris row count: ${cnt}")
+ cnt.size() == 1 && cnt.get(0).get(0) == totalRows
+ }
+ )
+ } catch (Exception ex) {
+ samplerStop.set(true)
+ def showjob = sql """select * from jobs("type"="insert") where Name='${jobName}'"""
+ def showtask = sql """select * from tasks("type"="insert") where JobName='${jobName}'"""
+ log.info("show job: " + showjob)
+ log.info("show task: " + showtask)
+ throw ex
+ }
+ samplerStop.set(true)
+ sampler.join(5000)
+ log.info("chunk_list lengths observed during snapshot: ${lengthsSeen.toSorted()}")
+
+ // >=3 distinct lengths = chunk_list grew in batched UPSERTs; one-shot would show only one.
+ assert lengthsSeen.size() >= 3 :
+ "uneven splitting should produce chunk_list in batches (>=3 distinct lengths)," +
+ " saw only ${lengthsSeen.toSorted()} — implies one-shot splitting"
+
+ // DISTINCT count guards against duplicates from chunk re-cut / RPC retry.
+ def doneDistinct = sql """SELECT COUNT(DISTINCT id) FROM ${currentDb}.${table1}"""
+ assert doneDistinct.get(0).get(0) == totalRows :
+ "DISTINCT pk count ${doneDistinct.get(0).get(0)} != ${totalRows} — chunks may have re-cut"
+
+ def loadStat0 = parseJson(sql("""
+ select loadStatistic from jobs("type"="insert") where Name='${jobName}'
+ """).get(0).get(0))
+ log.info("loadStat after uneven snapshot: ${loadStat0}")
+ assert loadStat0.scannedRows == totalRows
+
+ // Verify binlog phase still drains after uneven snapshot.
+ connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") {
+ sql """INSERT INTO ${pgDB}.${pgSchema}.${table1} (id, name, age) VALUES ('k_99999', 'incr', 999);"""
+ sql """UPDATE ${pgDB}.${pgSchema}.${table1} SET age = 8888 WHERE id = 'k_00001';"""
+ sql """DELETE FROM ${pgDB}.${pgSchema}.${table1} WHERE id = 'k_00002';"""
+ }
+
+ try {
+ Awaitility.await().atMost(120, SECONDS)
+ .pollInterval(2, SECONDS).until(
+ {
+ def res = sql """SELECT
+ COUNT(*),
+ SUM(CASE WHEN id='k_00001' AND age=8888 THEN 1 ELSE 0 END),
+ SUM(CASE WHEN id='k_99999' THEN 1 ELSE 0 END),
+ SUM(CASE WHEN id='k_00002' THEN 1 ELSE 0 END)
+ FROM ${currentDb}.${table1}"""
+ log.info("binlog state: ${res}")
+ res.size() == 1
+ && res.get(0).get(0) == totalRows
+ && res.get(0).get(1) == 1
+ && res.get(0).get(2) == 1
+ && res.get(0).get(3) == 0
+ }
+ )
+ } catch (Exception ex) {
+ def showjob = sql """select * from jobs("type"="insert") where Name='${jobName}'"""
+ log.info("show job: " + showjob)
+ throw ex
+ }
+
+ def jobInfo = sql """select status from jobs("type"="insert") where Name='${jobName}'"""
+ assert jobInfo.get(0).get(0) == "RUNNING"
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ def jobCountRsp = sql """select count(1) from jobs("type"="insert") where Name='${jobName}'"""
+ assert jobCountRsp.get(0).get(0) == 0
+ }
+}
diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_async_split_uneven_restart_fe.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_async_split_uneven_restart_fe.groovy
new file mode 100644
index 00000000000..97a7e560673
--- /dev/null
+++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_async_split_uneven_restart_fe.groovy
@@ -0,0 +1,207 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+import org.apache.doris.regression.suite.ClusterOptions
+import org.awaitility.Awaitility
+
+import static java.util.concurrent.TimeUnit.SECONDS
+
+// Restarts FE while the uneven splitter is mid-snapshot. Combines the slow uneven
+// path (VARCHAR PK -> splitOneUnevenlySizedChunk) with editlog replay to verify
+// cdcSplitProgress resumes from the system table — no rows lost, no duplicates
+// from re-cut chunks. batch_size is shrunk so the splitter must make multiple
+// RPCs, otherwise the default would let cdc_client finish all chunks in one shot
+// and the restart would only test task-dispatch resume, not split resume.
+suite("test_streaming_postgres_job_async_split_uneven_restart_fe",
+ "docker,pg,external_docker,external_docker_pg,nondatalake") {
+ def jobName = "test_streaming_postgres_job_async_split_uneven_restart_fe"
+ def options = new ClusterOptions()
+ options.setFeNum(1)
+ options.cloudMode = null
+
+ docker(options) {
+ def currentDb = (sql "select database()")[0][0]
+ def table1 = "user_info_pg_async_split_uneven_restart"
+ def pgDB = "postgres"
+ def pgSchema = "cdc_test"
+ def pgUser = "postgres"
+ def pgPassword = "123456"
+ def totalRows = 200
+ int expectedChunks = (int) Math.ceil(totalRows / 5.0) // 100
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ sql """drop table if exists ${currentDb}.${table1} force"""
+
+ String enabled = context.config.otherConfigs.get("enableJdbcTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String pg_port = context.config.otherConfigs.get("pg_14_port");
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String s3_endpoint = getS3Endpoint()
+ String bucket = getS3BucketName()
+ String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/postgresql-42.5.0.jar"
+
+ // Shrink batch_size so 100 chunks need ~20 RPCs — required to keep the
+ // splitter mid-flight when restart fires. With default 100, cdc_client
+ // returns all chunks in one RPC and the restart only validates task
+ // dispatch resume, not cdcSplitProgress replay.
+ def origBatchSize = sql("""ADMIN SHOW FRONTEND CONFIG LIKE 'streaming_cdc_fetch_splits_batch_size'""")
+ .get(0).get(1)
+ sql """ADMIN SET FRONTEND CONFIG ('streaming_cdc_fetch_splits_batch_size' = '5')"""
+
+ try {
+ // VARCHAR PK forces the uneven splitter; split_size=5 + parallelism=1 keeps
+ // consumption slow enough that we can restart while still mid-snapshot.
+ connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") {
+ sql """DROP TABLE IF EXISTS ${pgDB}.${pgSchema}.${table1}"""
+ sql """CREATE TABLE ${pgDB}.${pgSchema}.${table1} (
+ "id" varchar(20) PRIMARY KEY,
+ "name" varchar(200)
+ )"""
+ StringBuilder sb = new StringBuilder()
+ sb.append("INSERT INTO ${pgDB}.${pgSchema}.${table1} (id, name) VALUES ")
+ for (int i = 1; i <= totalRows; i++) {
+ if (i > 1) sb.append(", ")
+ String key = "k_" + String.format("%05d", i)
+ sb.append("('${key}', 'name_${i}')")
+ }
+ sql sb.toString()
+ }
+
+ sql """CREATE JOB ${jobName}
+ ON STREAMING
+ FROM POSTGRES (
+ "jdbc_url" = "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "org.postgresql.Driver",
+ "user" = "${pgUser}",
+ "password" = "${pgPassword}",
+ "database" = "${pgDB}",
+ "schema" = "${pgSchema}",
+ "include_tables" = "${table1}",
+ "offset" = "initial",
+ "snapshot_split_size" = "5",
+ "snapshot_parallelism" = "1"
+ )
+ TO DATABASE ${currentDb} (
+ "table.create.properties.replication_num" = "1"
+ )
+ """
+
+ def jobIdRow = sql """select Id from jobs("type"="insert") where Name='${jobName}'"""
+ assert jobIdRow.size() == 1
+ def jobId = jobIdRow.get(0).get(0).toString()
+
+ // Wait until at least 3 tasks succeeded AND splitter is still mid-flight
+ // (chunk_list < expectedChunks). Both conditions: task dispatch has made
+ // observable progress AND cdcSplitProgress has work left to replay.
+ try {
+ Awaitility.await().atMost(180, SECONDS)
+ .pollInterval(1, SECONDS).until(
+ {
+ def succeed = sql """select SucceedTaskCount from jobs("type"="insert") where Name = '${jobName}'"""
+ int chunkLen = -1
+ try {
+ def chunkRow = sql """SELECT json_length(chunk_list)
+ FROM internal.__internal_schema.streaming_job_meta
+ WHERE job_id='${jobId}'"""
+ if (chunkRow.size() > 0 && chunkRow.get(0).get(0) != null) {
+ chunkLen = chunkRow.get(0).get(0) as int
+ }
+ } catch (Throwable ignored) { /* meta table is lazy-created on first UPSERT */ }
+ log.info("pre-restart succeed=${succeed} chunkLen=${chunkLen}")
+ succeed.size() == 1
+ && Integer.parseInt(succeed.get(0).get(0).toString()) >= 3
+ && chunkLen > 0 && chunkLen < expectedChunks
+ }
+ )
+ } catch (Exception ex) {
+ def showjob = sql """select * from jobs("type"="insert") where Name='${jobName}'"""
+ log.info("show job: " + showjob)
+ throw ex
+ }
+
+ def rowsBefore = sql """SELECT COUNT(*) FROM ${currentDb}.${table1}"""
+ def chunkLenBefore = (sql """SELECT json_length(chunk_list)
+ FROM internal.__internal_schema.streaming_job_meta
+ WHERE job_id='${jobId}'""").get(0).get(0) as int
+ log.info("before restart: rows=${rowsBefore} chunkLen=${chunkLenBefore}")
+ assert rowsBefore.get(0).get(0) < totalRows :
+ "uneven snapshot finished too fast (${rowsBefore.get(0).get(0)} rows) — restart wouldn't be mid-snapshot"
+ assert chunkLenBefore < expectedChunks :
+ "splitter already finished (chunk_list=${chunkLenBefore}/${expectedChunks}) — restart can't exercise cdcSplitProgress replay"
+
+ cluster.restartFrontends()
+ sleep(30000)
+ context.reconnectFe()
+
+ try {
+ Awaitility.await().atMost(600, SECONDS)
+ .pollInterval(2, SECONDS).until(
+ {
+ def cnt = sql """SELECT COUNT(*) FROM ${currentDb}.${table1}"""
+ log.info("doris row count after restart: ${cnt}")
+ cnt.size() == 1 && cnt.get(0).get(0) == totalRows
+ }
+ )
+ } catch (Exception ex) {
+ def showjob = sql """select * from jobs("type"="insert") where Name='${jobName}'"""
+ def showtask = sql """select * from tasks("type"="insert") where JobName='${jobName}'"""
+ log.info("show job: " + showjob)
+ log.info("show task: " + showtask)
+ throw ex
+ }
+
+ def distinctCnt = sql """SELECT COUNT(DISTINCT id) FROM ${currentDb}.${table1}"""
+ assert distinctCnt.get(0).get(0) == totalRows :
+ "row count matches but DISTINCT differs — chunks re-cut on resume"
+
+ // Spot-check both ends of the PK range; if any chunk was lost on the boundary
+ // the missing rows would cluster at one end.
+ def lastKey = "k_" + String.format("%05d", totalRows)
+ def firstRow = sql """SELECT name FROM ${currentDb}.${table1} WHERE id='k_00001'"""
+ def lastRow = sql """SELECT name FROM ${currentDb}.${table1} WHERE id='${lastKey}'"""
+ assert firstRow.size() == 1 && firstRow.get(0).get(0) == 'name_1'
+ assert lastRow.size() == 1 && lastRow.get(0).get(0) == "name_${totalRows}"
+
+ def status = sql """select status from jobs("type"="insert") where Name='${jobName}'"""
+ assert status.get(0).get(0) == "RUNNING"
+
+ // Binlog phase after resume.
+ connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") {
+ sql """INSERT INTO ${pgDB}.${pgSchema}.${table1} (id, name) VALUES ('k_99999', 'post_restart');"""
+ }
+ try {
+ Awaitility.await().atMost(120, SECONDS)
+ .pollInterval(2, SECONDS).until(
+ {
+ def r = sql """SELECT count(*) FROM ${currentDb}.${table1} WHERE id='k_99999'"""
+ r.size() == 1 && r.get(0).get(0) == 1
+ }
+ )
+ } catch (Exception ex) {
+ log.info("job: " + (sql """select * from jobs("type"="insert") where Name='${jobName}'"""))
+ throw ex
+ }
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ } finally {
+ sql """ADMIN SET FRONTEND CONFIG ('streaming_cdc_fetch_splits_batch_size' = '${origBatchSize}')"""
+ }
+ }
+ }
+}
diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_binlog_restart_fe.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_binlog_restart_fe.groovy
new file mode 100644
index 00000000000..a4960b9c4f6
--- /dev/null
+++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_binlog_restart_fe.groovy
@@ -0,0 +1,191 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+import org.apache.doris.regression.suite.ClusterOptions
+import org.awaitility.Awaitility
+
+import static java.util.concurrent.TimeUnit.SECONDS
+
+// Restart FE AFTER snapshot fully transitioned to binlog. Replay goes through the
+// binlogOffsetPersist non-empty branch, which leaves finishedSplits/remainingSplits
+// empty. Without the noMoreSplits() guard on currentOffset=BinlogSplit, the scheduler
+// next tick would (mis)compute every cachedSyncTables entry as untouched and call
+// advanceSplits() to re-cut snapshot chunks, duplicating rows.
+suite("test_streaming_postgres_job_binlog_restart_fe",
+ "docker,pg,external_docker,external_docker_pg,nondatalake") {
+ def jobName = "test_streaming_postgres_job_binlog_restart_fe"
+ def options = new ClusterOptions()
+ options.setFeNum(1)
+ options.cloudMode = null
+
+ docker(options) {
+ def currentDb = (sql "select database()")[0][0]
+ def table1 = "user_info_pg_binlog_restart_fe"
+ def pgDB = "postgres"
+ def pgSchema = "cdc_test"
+ def pgUser = "postgres"
+ def pgPassword = "123456"
+ def totalRows = 50
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ sql """drop table if exists ${currentDb}.${table1} force"""
+
+ String enabled = context.config.otherConfigs.get("enableJdbcTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String pg_port = context.config.otherConfigs.get("pg_14_port");
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String s3_endpoint = getS3Endpoint()
+ String bucket = getS3BucketName()
+ String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/postgresql-42.5.0.jar"
+
+ connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") {
+ sql """DROP TABLE IF EXISTS ${pgDB}.${pgSchema}.${table1}"""
+ sql """CREATE TABLE ${pgDB}.${pgSchema}.${table1} (
+ "id" varchar(20) PRIMARY KEY,
+ "name" varchar(200)
+ )"""
+ StringBuilder sb = new StringBuilder()
+ sb.append("INSERT INTO ${pgDB}.${pgSchema}.${table1} (id, name) VALUES ")
+ for (int i = 1; i <= totalRows; i++) {
+ if (i > 1) sb.append(", ")
+ String key = "k_" + String.format("%05d", i)
+ sb.append("('${key}', 'name_${i}')")
+ }
+ sql sb.toString()
+ }
+
+ sql """CREATE JOB ${jobName}
+ ON STREAMING
+ FROM POSTGRES (
+ "jdbc_url" = "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "org.postgresql.Driver",
+ "user" = "${pgUser}",
+ "password" = "${pgPassword}",
+ "database" = "${pgDB}",
+ "schema" = "${pgSchema}",
+ "include_tables" = "${table1}",
+ "offset" = "initial",
+ "snapshot_split_size" = "10",
+ "snapshot_parallelism" = "1"
+ )
+ TO DATABASE ${currentDb} (
+ "table.create.properties.replication_num" = "1"
+ )
+ """
+
+ def jobIdRow = sql """select Id from jobs("type"="insert") where Name='${jobName}'"""
+ assert jobIdRow.size() == 1
+ def jobId = jobIdRow.get(0).get(0).toString()
+
+ // Step 1: wait until snapshot fully drained into Doris.
+ Awaitility.await().atMost(180, SECONDS).pollInterval(1, SECONDS).until(
+ {
+ def cnt = sql """SELECT COUNT(*) FROM ${currentDb}.${table1}"""
+ cnt.size() == 1 && cnt.get(0).get(0) == totalRows
+ }
+ )
+
+ // Step 2: insert one binlog row so updateOffset commits a BinlogSplit ->
+ // binlogOffsetPersist gets written to EditLog. This is the precondition
+ // for the replay path we want to exercise.
+ connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") {
+ sql """INSERT INTO ${pgDB}.${pgSchema}.${table1} (id, name) VALUES ('k_99999', 'pre_restart')"""
+ }
+ Awaitility.await().atMost(120, SECONDS).pollInterval(1, SECONDS).until(
+ {
+ def r = sql """SELECT count(*) FROM ${currentDb}.${table1} WHERE id='k_99999'"""
+ r.size() == 1 && r.get(0).get(0) == 1
+ }
+ )
+
+ // Snapshot of state before restart — used to detect re-cut / re-read.
+ def rowsBefore = (sql """SELECT COUNT(*) FROM ${currentDb}.${table1}""").get(0).get(0)
+ int chunkLenBefore = -1
+ String chunkListBefore = null
+ try {
+ def chunkRow = sql """SELECT json_length(chunk_list), chunk_list
+ FROM internal.__internal_schema.streaming_job_meta
+ WHERE job_id='${jobId}'"""
+ if (chunkRow.size() > 0 && chunkRow.get(0).get(0) != null) {
+ chunkLenBefore = chunkRow.get(0).get(0) as int
+ chunkListBefore = chunkRow.get(0).get(1).toString()
+ }
+ } catch (Throwable ignored) { /* meta table optional */ }
+ log.info("before restart: rows=${rowsBefore} chunkLen=${chunkLenBefore}")
+
+ // Step 3: restart FE, then give scheduler several ticks to misbehave (if buggy).
+ cluster.restartFrontends()
+ sleep(30000)
+ context.reconnectFe()
+ sleep(15000)
+
+ // Step 4: state must be stable — no extra rows, no duplicates, chunk_list unchanged.
+ // chunk_list content (not size) is the key signal: a buggy advanceSplits in
+ // binlog phase re-cuts chunks and UPSERTs them under the same (job_id, split_id)
+ // keys, so size stays the same but the rows themselves get rewritten.
+ def rowsAfter = (sql """SELECT COUNT(*) FROM ${currentDb}.${table1}""").get(0).get(0)
+ def distinctAfter = (sql """SELECT COUNT(DISTINCT id) FROM ${currentDb}.${table1}""").get(0).get(0)
+ int chunkLenAfter = -1
+ String chunkListAfter = null
+ try {
+ def chunkRow = sql """SELECT json_length(chunk_list), chunk_list
+ FROM internal.__internal_schema.streaming_job_meta
+ WHERE job_id='${jobId}'"""
+ if (chunkRow.size() > 0 && chunkRow.get(0).get(0) != null) {
+ chunkLenAfter = chunkRow.get(0).get(0) as int
+ chunkListAfter = chunkRow.get(0).get(1).toString()
+ }
+ } catch (Throwable ignored) { /* meta table optional */ }
+ log.info("after restart: rows=${rowsAfter} distinct=${distinctAfter} chunkLen=${chunkLenAfter}")
+
+ assert rowsAfter == rowsBefore :
+ "row count grew after restart (${rowsBefore} -> ${rowsAfter}) — " +
+ "scheduler re-cut snapshot in binlog phase"
+ assert distinctAfter == rowsAfter :
+ "duplicate ids after restart (rows=${rowsAfter} distinct=${distinctAfter})"
+ if (chunkLenBefore >= 0) {
+ assert chunkLenAfter == chunkLenBefore :
+ "chunk_list grew after restart (${chunkLenBefore} -> ${chunkLenAfter}) — " +
+ "advanceSplits ran in binlog phase"
+ }
+ if (chunkListBefore != null && chunkListAfter != null) {
+ assert chunkListBefore == chunkListAfter :
+ "chunk_list content rewritten after restart — advanceSplits ran in binlog phase.\n" +
+ " before: ${chunkListBefore}\n" +
+ " after: ${chunkListAfter}"
+ }
+
+ // Step 5: binlog must still work after restart.
+ connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") {
+ sql """INSERT INTO ${pgDB}.${pgSchema}.${table1} (id, name) VALUES ('k_99998', 'post_restart')"""
+ }
+ Awaitility.await().atMost(120, SECONDS).pollInterval(2, SECONDS).until(
+ {
+ def r = sql """SELECT count(*) FROM ${currentDb}.${table1} WHERE id='k_99998'"""
+ r.size() == 1 && r.get(0).get(0) == 1
+ }
+ )
+
+ def status = sql """select status from jobs("type"="insert") where Name='${jobName}'"""
+ assert status.get(0).get(0) == "RUNNING"
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ }
+ }
+}
diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_special_offset.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_special_offset.groovy
index 1b64f111859..03af48213aa 100644
--- a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_special_offset.groovy
+++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_special_offset.groovy
@@ -76,10 +76,21 @@ suite("test_streaming_postgres_job_special_offset", "p0,external,pg,external_doc
connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") {
sql """INSERT INTO ${pgDB}.${pgSchema}.${table1} VALUES (3, 'charlie')"""
}
+ // Mode-gate check: only the post-CREATE INSERT must arrive; pre-existing (1, 2) are skipped.
Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({
- def result = sql """SELECT count(*) FROM ${currentDb}.${table1}"""
- return result[0][0] >= 1
+ def result = sql """SELECT count(*) FROM ${currentDb}.${table1} WHERE id = 3"""
+ return result[0][0] == 1
})
+ def preCount = sql """SELECT count(*) FROM ${currentDb}.${table1} WHERE id IN (1, 2)"""
+ assert preCount.get(0).get(0) == 0 :
+ "offset=latest should skip pre-existing rows, but found ${preCount.get(0).get(0)} of them"
+ def loadStat = parseJson(sql("""
+ select loadStatistic from jobs("type"="insert") where Name='${jobName}'
+ """).get(0).get(0))
+ log.info("loadStat: ${loadStat}")
+ assert loadStat.scannedRows == 1 :
+ "expected scannedRows=1 (only the binlog INSERT), got ${loadStat.scannedRows}"
+
sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
sql """drop table if exists ${currentDb}.${table1} force"""
diff --git a/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_postgres_async_split.groovy b/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_postgres_async_split.groovy
new file mode 100644
index 00000000000..e82f6717245
--- /dev/null
+++ b/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_postgres_async_split.groovy
@@ -0,0 +1,193 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+import org.awaitility.Awaitility
+
+import java.util.concurrent.CopyOnWriteArraySet
+import java.util.concurrent.atomic.AtomicBoolean
+
+import static java.util.concurrent.TimeUnit.SECONDS
+
+// cdc_stream TVF path + uneven splitter (VARCHAR PK). With 1500 rows / split_size=5
+// = 300 chunks > default batch_size=100, the splitter must take ~3 RPCs so
+// streaming_job_meta.chunk_list grows in steps — one-shot splitting would land the
+// full list in a single UPSERT. We avoid mutating the global FE config because
+// concurrent cdc cases that also adjust batch_size race with us.
+suite("test_streaming_job_cdc_stream_postgres_async_split",
+ "p0,external,pg,external_docker,external_docker_pg,nondatalake") {
+ def jobName = "test_streaming_job_cdc_stream_postgres_async_split"
+ def currentDb = (sql "select database()")[0][0]
+ def dorisTable = "test_streaming_job_cdc_stream_postgres_async_split_tbl"
+ def pgDB = "postgres"
+ def pgSchema = "cdc_test"
+ def pgUser = "postgres"
+ def pgPassword = "123456"
+ def pgTable = "test_streaming_job_cdc_stream_postgres_async_split_src"
+ def totalRows = 1500
+ int expectedChunks = (int) Math.ceil(totalRows / 5.0) // 300 chunks at split_size=5
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ sql """drop table if exists ${currentDb}.${dorisTable} force"""
+
+ sql """
+ CREATE TABLE IF NOT EXISTS ${currentDb}.${dorisTable} (
+ `id` varchar(20) NULL,
+ `name` varchar(200) NULL,
+ `age` int NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS AUTO
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+ """
+
+ String enabled = context.config.otherConfigs.get("enableJdbcTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String pg_port = context.config.otherConfigs.get("pg_14_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String s3_endpoint = getS3Endpoint()
+ String bucket = getS3BucketName()
+ String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/postgresql-42.5.0.jar"
+
+ // VARCHAR PK bypasses isEvenlySplitColumn() -> uneven splitter path.
+ connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") {
+ sql """DROP TABLE IF EXISTS ${pgDB}.${pgSchema}.${pgTable}"""
+ sql """CREATE TABLE ${pgDB}.${pgSchema}.${pgTable} (
+ "id" varchar(20) PRIMARY KEY,
+ "name" varchar(200),
+ "age" int4
+ )"""
+ StringBuilder sb = new StringBuilder()
+ sb.append("INSERT INTO ${pgDB}.${pgSchema}.${pgTable} (id, name, age) VALUES ")
+ for (int i = 1; i <= totalRows; i++) {
+ if (i > 1) sb.append(", ")
+ String key = "k_" + String.format("%05d", i)
+ sb.append("('${key}', 'name_${i}', ${i})")
+ }
+ sql sb.toString()
+ }
+
+ sql """
+ CREATE JOB ${jobName}
+ ON STREAMING DO INSERT INTO ${currentDb}.${dorisTable} (id, name, age)
+ SELECT id, name, age FROM cdc_stream(
+ "type" = "postgres",
+ "jdbc_url" = "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "org.postgresql.Driver",
+ "user" = "${pgUser}",
+ "password" = "${pgPassword}",
+ "database" = "${pgDB}",
+ "schema" = "${pgSchema}",
+ "table" = "${pgTable}",
+ "offset" = "initial",
+ "snapshot_split_size" = "5",
+ "snapshot_parallelism" = "2"
+ )
+ """
+
+ def jobIdRow = sql """select Id from jobs("type"="insert") where Name='${jobName}'"""
+ assert jobIdRow.size() == 1
+ def jobId = jobIdRow.get(0).get(0).toString()
+
+ // Background sampler: poll streaming_job_meta.chunk_list length every 150ms while
+ // the splitter runs. With default batch_size=100 and 300 chunks the splitter does
+ // ~3 RPCs so we should observe >=3 distinct lengths (100, 200, 300).
+ def lengthsSeen = new CopyOnWriteArraySet<Integer>()
+ def samplerStop = new AtomicBoolean(false)
+ def sampler = Thread.start {
+ int lastSeen = -1
+ while (!samplerStop.get()) {
+ try {
+ def r = sql """SELECT json_length(chunk_list)
+ FROM internal.__internal_schema.streaming_job_meta
+ WHERE job_id='${jobId}'"""
+ if (r.size() > 0 && r.get(0).get(0) != null) {
+ int len = r.get(0).get(0) as int
+ lengthsSeen.add(len)
+ if (len != lastSeen) {
+ log.info("sampler: chunk_list length=${len}")
+ lastSeen = len
+ }
+ // Assertion only needs >=3 distinct lengths; stop polling once satisfied
+ // to keep the case log slim during the remaining snapshot consumption.
+ if (lengthsSeen.size() >= 3) break
+ }
+ } catch (Throwable ignored) { /* meta table may not have row yet */ }
+ try { Thread.sleep(150) } catch (InterruptedException ie) { break }
+ }
+ }
+
+ try {
+ Awaitility.await().atMost(600, SECONDS)
+ .pollInterval(2, SECONDS).until(
+ {
+ def cnt = sql """SELECT COUNT(*) FROM ${currentDb}.${dorisTable}"""
+ log.info("doris row count: ${cnt}")
+ cnt.size() == 1 && cnt.get(0).get(0) == totalRows
+ }
+ )
+ } catch (Exception ex) {
+ samplerStop.set(true)
+ def showjob = sql """select * from jobs("type"="insert") where Name='${jobName}'"""
+ def showtask = sql """select * from tasks("type"="insert") where JobName='${jobName}'"""
+ log.info("show job: " + showjob)
+ log.info("show task: " + showtask)
+ throw ex
+ }
+ samplerStop.set(true)
+ sampler.join(5000)
+ log.info("chunk_list lengths observed during snapshot: ${lengthsSeen.toSorted()}")
+
+ // >=3 distinct lengths = chunk_list grew in batched UPSERTs; one-shot would show only one.
+ assert lengthsSeen.size() >= 3 :
+ "TVF uneven splitting should produce chunk_list in batches (>=3 distinct lengths)," +
+ " saw only ${lengthsSeen.toSorted()} — implies one-shot splitting"
+
+ def doneDistinct = sql """SELECT COUNT(DISTINCT id) FROM ${currentDb}.${dorisTable}"""
+ assert doneDistinct.get(0).get(0) == totalRows :
+ "DISTINCT pk count ${doneDistinct.get(0).get(0)} != ${totalRows} — chunks may have re-cut"
+
+ // Verify binlog phase: INSERT propagates after snapshot. UPDATE/DELETE skipped
+ // because dorisTable is DUPLICATE KEY (append-only) and would yield duplicate rows.
+ connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") {
+ sql """INSERT INTO ${pgDB}.${pgSchema}.${pgTable} (id, name, age) VALUES ('k_99999', 'incr', 999);"""
+ }
+
+ try {
+ Awaitility.await().atMost(120, SECONDS)
+ .pollInterval(2, SECONDS).until(
+ {
+ def r = sql """SELECT count(*) FROM ${currentDb}.${dorisTable} WHERE id='k_99999'"""
+ log.info("binlog state: ${r}")
+ r.size() == 1 && r.get(0).get(0) == 1
+ }
+ )
+ } catch (Exception ex) {
+ def showjob = sql """select * from jobs("type"="insert") where Name='${jobName}'"""
+ log.info("show job: " + showjob)
+ throw ex
+ }
+
+ def jobInfo = sql """select status from jobs("type"="insert") where Name='${jobName}'"""
+ assert jobInfo.get(0).get(0) == "RUNNING"
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ def jobCountRsp = sql """select count(1) from jobs("type"="insert") where Name='${jobName}'"""
+ assert jobCountRsp.get(0).get(0) == 0
+ }
+}
diff --git a/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_postgres_async_split_restart_fe.groovy b/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_postgres_async_split_restart_fe.groovy
new file mode 100644
index 00000000000..330e1338602
--- /dev/null
+++ b/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_postgres_async_split_restart_fe.groovy
@@ -0,0 +1,218 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+import org.apache.doris.regression.suite.ClusterOptions
+import org.awaitility.Awaitility
+
+import static java.util.concurrent.TimeUnit.SECONDS
+
+// TVF (cdc_stream) variant of the uneven-restart-fe case. The TVF path has its own
+// JdbcTvfSourceOffsetProvider.replayIfNeed; this case verifies that after a mid-snapshot
+// FE restart, cachedSyncTables is restored, restored remainingSplits are consumed, AND
+// advanceSplitsIfNeed continues to fetch the next batch (i.e. snapshot is not truncated
+// to just the chunks already in the meta table).
+suite("test_streaming_job_cdc_stream_postgres_async_split_restart_fe",
+ "docker,pg,external_docker,external_docker_pg,nondatalake") {
+ def jobName = "test_streaming_job_cdc_stream_postgres_async_split_restart_fe"
+ def options = new ClusterOptions()
+ options.setFeNum(1)
+ options.cloudMode = null
+
+ docker(options) {
+ def currentDb = (sql "select database()")[0][0]
+ def dorisTable = "streaming_job_cdc_stream_postgres_async_split_restart_fe_tbl"
+ def pgDB = "postgres"
+ def pgSchema = "cdc_test"
+ def pgUser = "postgres"
+ def pgPassword = "123456"
+ def pgTable = "streaming_job_cdc_stream_postgres_async_split_restart_fe_src"
+ def totalRows = 200
+ int expectedChunks = (int) Math.ceil(totalRows / 5.0) // 100
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ sql """drop table if exists ${currentDb}.${dorisTable} force"""
+
+ sql """
+ CREATE TABLE IF NOT EXISTS ${currentDb}.${dorisTable} (
+ `id` varchar(20) NULL,
+ `name` varchar(200) NULL,
+ `age` int NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS AUTO
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+ """
+
+ String enabled = context.config.otherConfigs.get("enableJdbcTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String pg_port = context.config.otherConfigs.get("pg_14_port");
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String s3_endpoint = getS3Endpoint()
+ String bucket = getS3BucketName()
+ String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/postgresql-42.5.0.jar"
+
+ // Shrink batch_size so 100 chunks need ~20 RPCs — required so cdcSplitProgress
+ // is genuinely mid-flight when restart fires (default 100 = one-shot split).
+ def origBatchSize = sql("""ADMIN SHOW FRONTEND CONFIG LIKE 'streaming_cdc_fetch_splits_batch_size'""")
+ .get(0).get(1)
+ sql """ADMIN SET FRONTEND CONFIG ('streaming_cdc_fetch_splits_batch_size' = '5')"""
+
+ try {
+ connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") {
+ sql """DROP TABLE IF EXISTS ${pgDB}.${pgSchema}.${pgTable}"""
+ sql """CREATE TABLE ${pgDB}.${pgSchema}.${pgTable} (
+ "id" varchar(20) PRIMARY KEY,
+ "name" varchar(200),
+ "age" int4
+ )"""
+ StringBuilder sb = new StringBuilder()
+ sb.append("INSERT INTO ${pgDB}.${pgSchema}.${pgTable} (id, name, age) VALUES ")
+ for (int i = 1; i <= totalRows; i++) {
+ if (i > 1) sb.append(", ")
+ String key = "k_" + String.format("%05d", i)
+ sb.append("('${key}', 'name_${i}', ${i})")
+ }
+ sql sb.toString()
+ }
+
+ sql """
+ CREATE JOB ${jobName}
+ ON STREAMING DO INSERT INTO ${currentDb}.${dorisTable} (id, name, age)
+ SELECT id, name, age FROM cdc_stream(
+ "type" = "postgres",
+ "jdbc_url" = "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "org.postgresql.Driver",
+ "user" = "${pgUser}",
+ "password" = "${pgPassword}",
+ "database" = "${pgDB}",
+ "schema" = "${pgSchema}",
+ "table" = "${pgTable}",
+ "offset" = "initial",
+ "snapshot_split_size" = "5",
+ "snapshot_parallelism" = "1"
+ )
+ """
+
+ def jobIdRow = sql """select Id from jobs("type"="insert") where Name='${jobName}'"""
+ assert jobIdRow.size() == 1
+ def jobId = jobIdRow.get(0).get(0).toString()
+
+ // Wait until tasks have committed AND splitter is still mid-flight.
+ try {
+ Awaitility.await().atMost(180, SECONDS)
+ .pollInterval(1, SECONDS).until(
+ {
+ def succeed = sql """select SucceedTaskCount from jobs("type"="insert") where Name = '${jobName}'"""
+ int chunkLen = -1
+ try {
+ def chunkRow = sql """SELECT json_length(chunk_list)
+ FROM internal.__internal_schema.streaming_job_meta
+ WHERE job_id='${jobId}'"""
+ if (chunkRow.size() > 0 && chunkRow.get(0).get(0) != null) {
+ chunkLen = chunkRow.get(0).get(0) as int
+ }
+ } catch (Throwable ignored) { /* meta table is lazy-created on first UPSERT */ }
+ log.info("pre-restart succeed=${succeed} chunkLen=${chunkLen}")
+ succeed.size() == 1
+ && Integer.parseInt(succeed.get(0).get(0).toString()) >= 3
+ && chunkLen > 0 && chunkLen < expectedChunks
+ }
+ )
+ } catch (Exception ex) {
+ def showjob = sql """select * from jobs("type"="insert") where Name='${jobName}'"""
+ log.info("show job: " + showjob)
+ throw ex
+ }
+
+ def rowsBefore = sql """SELECT COUNT(*) FROM ${currentDb}.${dorisTable}"""
+ def chunkLenBefore = (sql """SELECT json_length(chunk_list)
+ FROM internal.__internal_schema.streaming_job_meta
+ WHERE job_id='${jobId}'""").get(0).get(0) as int
+ log.info("before restart: rows=${rowsBefore} chunkLen=${chunkLenBefore}")
+ assert rowsBefore.get(0).get(0) < totalRows :
+ "TVF snapshot finished too fast (${rowsBefore.get(0).get(0)} rows) — restart wouldn't be mid-snapshot"
+ assert chunkLenBefore < expectedChunks :
+ "splitter already finished (chunk_list=${chunkLenBefore}/${expectedChunks}) — restart can't exercise replay"
+
+ cluster.restartFrontends()
+ sleep(30000)
+ context.reconnectFe()
+
+ // After restart, snapshot must complete — verifies cachedSyncTables was restored
+ // by replayIfNeed and advanceSplitsIfNeed continues fetching beyond the chunks
+ // already in the meta table at restart time.
+ try {
+ Awaitility.await().atMost(600, SECONDS)
+ .pollInterval(2, SECONDS).until(
+ {
+ def cnt = sql """SELECT COUNT(*) FROM ${currentDb}.${dorisTable}"""
+ log.info("doris row count after restart: ${cnt}")
+ cnt.size() == 1 && cnt.get(0).get(0) == totalRows
+ }
+ )
+ } catch (Exception ex) {
+ def showjob = sql """select * from jobs("type"="insert") where Name='${jobName}'"""
+ def showtask = sql """select * from tasks("type"="insert") where JobName='${jobName}'"""
+ log.info("show job: " + showjob)
+ log.info("show task: " + showtask)
+ throw ex
+ }
+
+ def distinctCnt = sql """SELECT COUNT(DISTINCT id) FROM ${currentDb}.${dorisTable}"""
+ assert distinctCnt.get(0).get(0) == totalRows :
+ "row count matches but DISTINCT differs — chunks re-cut on resume"
+
+ // Final chunk_list must have grown past pre-restart size — direct evidence that
+ // advanceSplits continued fetching after restart (not truncated to meta snapshot).
+ def chunkLenAfter = (sql """SELECT json_length(chunk_list)
+ FROM internal.__internal_schema.streaming_job_meta
+ WHERE job_id='${jobId}'""").get(0).get(0) as int
+ log.info("after restart: chunkLen=${chunkLenAfter}")
+ // Uneven sampler re-runs after restart, so total chunk count is non-deterministic.
+ assert chunkLenAfter > chunkLenBefore :
+ "chunk_list did not grow after restart (${chunkLenBefore} -> ${chunkLenAfter}) — " +
+ "snapshot was truncated to pre-restart state"
+
+ def status = sql """select status from jobs("type"="insert") where Name='${jobName}'"""
+ assert status.get(0).get(0) == "RUNNING"
+
+ // Binlog phase after resume must still work.
+ connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") {
+ sql """INSERT INTO ${pgDB}.${pgSchema}.${pgTable} (id, name, age) VALUES ('k_99999', 'post_restart', 999);"""
+ }
+ try {
+ Awaitility.await().atMost(120, SECONDS)
+ .pollInterval(2, SECONDS).until(
+ {
+ def r = sql """SELECT count(*) FROM ${currentDb}.${dorisTable} WHERE id='k_99999'"""
+ r.size() == 1 && r.get(0).get(0) == 1
+ }
+ )
+ } catch (Exception ex) {
+ log.info("job: " + (sql """select * from jobs("type"="insert") where Name='${jobName}'"""))
+ throw ex
+ }
+
+ sql """DROP JOB IF EXISTS where jobname = '${jobName}'"""
+ } finally {
+ sql """ADMIN SET FRONTEND CONFIG ('streaming_cdc_fetch_splits_batch_size' = '${origBatchSize}')"""
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org