You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2025/11/12 14:22:38 UTC
(doris) branch branch-4.0 updated: branch-4.0: [Chore](job) remove unless code for streaming job #57873 (#57929)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new e052dd23fa4 branch-4.0: [Chore](job) remove unless code for streaming job #57873 (#57929)
e052dd23fa4 is described below
commit e052dd23fa47c381a1a48fd8d3430c6cf01c180a
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Wed Nov 12 22:22:29 2025 +0800
branch-4.0: [Chore](job) remove unless code for streaming job #57873 (#57929)
Cherry-picked from #57873
Co-authored-by: wudi <wu...@selectdb.com>
---
.../job/offset/s3/S3SourceOffsetProvider.java | 26 +++++++---------------
1 file changed, 8 insertions(+), 18 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/offset/s3/S3SourceOffsetProvider.java b/fe/fe-core/src/main/java/org/apache/doris/job/offset/s3/S3SourceOffsetProvider.java
index 15a961bb666..6e24b3d3680 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/offset/s3/S3SourceOffsetProvider.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/offset/s3/S3SourceOffsetProvider.java
@@ -80,27 +80,17 @@ public class S3SourceOffsetProvider implements SourceOffsetProvider {
String basePrefix = (lastSlash >= 0) ? prefix.substring(0, lastSlash + 1) : "";
String filePathBase = bucketBase + basePrefix;
String joined = rfiles.stream()
- .filter(name -> !name.equals(filePathBase)) // Single file case
.map(path -> path.getName().replace(filePathBase, ""))
.collect(Collectors.joining(","));
- if (joined.isEmpty()) {
- // base is a single file
- offset.setFileLists(filePathBase);
- String lastFile = rfiles.get(rfiles.size() - 1).getName().replace(bucketBase, "");
- offset.setStartFile(lastFile);
- offset.setEndFile(lastFile);
- } else {
- // base is dir
- String normalizedPrefix = basePrefix.endsWith("/")
- ? basePrefix.substring(0, basePrefix.length() - 1) : basePrefix;
- String finalFileLists = String.format("s3://%s/%s/{%s}", bucket, normalizedPrefix, joined);
- String beginFile = rfiles.get(0).getName().replace(bucketBase, "");
- String lastFile = rfiles.get(rfiles.size() - 1).getName().replace(bucketBase, "");
- offset.setFileLists(finalFileLists);
- offset.setStartFile(beginFile);
- offset.setEndFile(lastFile);
- }
+ String normalizedPrefix = basePrefix.endsWith("/")
+ ? basePrefix.substring(0, basePrefix.length() - 1) : basePrefix;
+ String finalFileLists = String.format("s3://%s/%s/{%s}", bucket, normalizedPrefix, joined);
+ String beginFile = rfiles.get(0).getName().replace(bucketBase, "");
+ String lastFile = rfiles.get(rfiles.size() - 1).getName().replace(bucketBase, "");
+ offset.setFileLists(finalFileLists);
+ offset.setStartFile(beginFile);
+ offset.setEndFile(lastFile);
offset.setFileNum(rfiles.size());
maxEndFile = globListResult.getMaxFile();
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org