You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by li...@apache.org on 2026/07/21 14:28:40 UTC

(seatunnel) branch dev updated: [Improve][Connector-V2][SftpFile] Defer binary file discovery (#11377)

This is an automated email from the ASF dual-hosted git repository.

davidzollo pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git


The following commit(s) were added to refs/heads/dev by this push:
     new 886c0fd13a [Improve][Connector-V2][SftpFile] Defer binary file discovery (#11377)
886c0fd13a is described below

commit 886c0fd13ad7e7e8a6e782299e7349dfa814ed7e
Author: Goutam Adwant <86...@users.noreply.github.com>
AuthorDate: Tue Jul 21 16:28:33 2026 +0200

    [Improve][Connector-V2][SftpFile] Defer binary file discovery (#11377)
    
    Signed-off-by: goutamadwant <wo...@gmail.com>
---
 .../file/config/BaseFileSourceConfig.java          |  78 +++++-
 .../MultipleTableFileSourceSplitEnumerator.java    |  21 +-
 ...MultipleTableFileSourceSplitEnumeratorTest.java | 306 +++++++++++++++++++--
 .../file/sftp/config/SFTPFileSourceConfig.java     |  10 +
 4 files changed, 376 insertions(+), 39 deletions(-)

diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseFileSourceConfig.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseFileSourceConfig.java
index 604cf90db3..08a0902f44 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseFileSourceConfig.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseFileSourceConfig.java
@@ -33,8 +33,11 @@ import org.apache.seatunnel.connectors.seatunnel.file.source.reader.ReadStrategy
 import org.apache.commons.collections4.CollectionUtils;
 
 import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
 
+import java.io.IOException;
 import java.io.Serializable;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -43,6 +46,7 @@ import java.util.function.Function;
 import java.util.stream.Collectors;
 
 @Getter
+@Slf4j
 public abstract class BaseFileSourceConfig implements Serializable {
 
     private static final long serialVersionUID = 1L;
@@ -53,6 +57,7 @@ public abstract class BaseFileSourceConfig implements Serializable {
     private final List<String> filePaths;
     private final ReadonlyConfig baseFileSourceConfig;
     private final CatalogTable catalogTableFromConfig;
+    private final boolean fileDiscoveryDeferred;
 
     public abstract HadoopConf getHadoopConfig();
 
@@ -63,20 +68,61 @@ public abstract class BaseFileSourceConfig implements Serializable {
         this.baseFileSourceConfig = readonlyConfig;
         this.fileFormat = readonlyConfig.get(FileBaseSourceOptions.FILE_FORMAT_TYPE);
         this.readStrategy = ReadStrategyFactory.of(readonlyConfig, getHadoopConfig());
+        this.fileDiscoveryDeferred = shouldDeferFileDiscovery(readonlyConfig);
         this.filePaths = parseFilePaths(readonlyConfig);
         this.catalogTableFromConfig = catalogTableFromConfig;
         this.catalogTable = parseCatalogTable(readonlyConfig);
     }
 
+    protected boolean shouldDeferFileDiscovery(ReadonlyConfig readonlyConfig) {
+        return false;
+    }
+
     private List<String> parseFilePaths(ReadonlyConfig readonlyConfig) {
-        if (readonlyConfig.get(FileBaseSourceOptions.DISCOVERY_MODE)
-                == FileDiscoveryMode.CONTINUOUS) {
+        if (readonlyConfig.get(FileBaseSourceOptions.DISCOVERY_MODE) == FileDiscoveryMode.CONTINUOUS
+                || fileDiscoveryDeferred) {
             return Collections.emptyList();
         }
-        String rootPath = null;
+        return discoverFilePaths();
+    }
+
+    public List<String> getFilePathsForSplitEnumerator() {
+        if (fileDiscoveryDeferred) {
+            ReadStrategy discoveryReadStrategy =
+                    ReadStrategyFactory.of(baseFileSourceConfig, getHadoopConfig());
+            try {
+                return discoverFilePaths(discoveryReadStrategy);
+            } finally {
+                closeDiscoveryReadStrategy(discoveryReadStrategy);
+            }
+        }
+        return filePaths;
+    }
+
+    private void closeDiscoveryReadStrategy(ReadStrategy discoveryReadStrategy) {
+        try {
+            discoveryReadStrategy.close();
+        } catch (IOException e) {
+            log.warn("Failed to close file discovery resources for plugin {}", getPluginName(), e);
+        }
+    }
+
+    private List<String> discoverFilePaths() {
+        return discoverFilePaths(readStrategy);
+    }
+
+    private List<String> discoverFilePaths(ReadStrategy discoveryReadStrategy) {
+        String rootPath = baseFileSourceConfig.get(FileBaseSourceOptions.FILE_PATH);
+        long startTime = System.currentTimeMillis();
         try {
-            rootPath = readonlyConfig.get(FileBaseSourceOptions.FILE_PATH);
-            return readStrategy.getFileNamesByPath(rootPath);
+            List<String> discoveredFilePaths = discoveryReadStrategy.getFileNamesByPath(rootPath);
+            log.info(
+                    "File source discovery finished: plugin={}, path={}, files={}, cost={}ms",
+                    getPluginName(),
+                    maskUriUserInfo(rootPath),
+                    discoveredFilePaths.size(),
+                    System.currentTimeMillis() - startTime);
+            return discoveredFilePaths;
         } catch (Exception ex) {
             String errorMsg = String.format("Get file list from this path [%s] failed", rootPath);
             throw new FileConnectorException(
@@ -127,6 +173,28 @@ public abstract class BaseFileSourceConfig implements Serializable {
         return readStrategy.getSeaTunnelRowTypeInfo(rootPath);
     }
 
+    private static String maskUriUserInfo(String rawPath) {
+        if (rawPath == null) {
+            return null;
+        }
+        try {
+            URI uri = URI.create(rawPath);
+            if (uri.getUserInfo() == null || uri.getAuthority() == null) {
+                return rawPath;
+            }
+            String maskedAuthority = uri.getAuthority().replace(uri.getUserInfo() + "@", "***@");
+            return new URI(
+                            uri.getScheme(),
+                            maskedAuthority,
+                            uri.getPath(),
+                            uri.getQuery(),
+                            uri.getFragment())
+                    .toString();
+        } catch (Exception e) {
+            return rawPath;
+        }
+    }
+
     private CatalogTable newCatalogTable(
             CatalogTable catalogTable, SeaTunnelRowType seaTunnelRowType) {
         TableSchema tableSchema = catalogTable.getTableSchema();
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/MultipleTableFileSourceSplitEnumerator.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/MultipleTableFileSourceSplitEnumerator.java
index b0c5dd2ad9..92acf52c80 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/MultipleTableFileSourceSplitEnumerator.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/MultipleTableFileSourceSplitEnumerator.java
@@ -48,7 +48,7 @@ public class MultipleTableFileSourceSplitEnumerator
     private final Context<FileSourceSplit> context;
     private final Set<FileSourceSplit> allSplit;
     private final Set<FileSourceSplit> assignedSplit;
-    private final Map<String, List<String>> filePathMap;
+    private final List<BaseFileSourceConfig> fileSourceConfigs;
     private final AtomicInteger assignCount = new AtomicInteger(0);
     private final Object lock = new Object();
     private final FileSplitStrategy fileSplitStrategy;
@@ -58,17 +58,7 @@ public class MultipleTableFileSourceSplitEnumerator
             BaseMultipleTableFileSourceConfig multipleTableFileSourceConfig,
             FileSplitStrategy fileSplitStrategy) {
         this.context = context;
-        this.filePathMap =
-                multipleTableFileSourceConfig.getFileSourceConfigs().stream()
-                        .collect(
-                                Collectors.toMap(
-                                        localFileSourceConfig ->
-                                                localFileSourceConfig
-                                                        .getCatalogTable()
-                                                        .getTableId()
-                                                        .toTablePath()
-                                                        .toString(),
-                                        BaseFileSourceConfig::getFilePaths));
+        this.fileSourceConfigs = multipleTableFileSourceConfig.getFileSourceConfigs();
         this.assignedSplit = new HashSet<>();
         this.allSplit = new TreeSet<>(Comparator.comparing(FileSourceSplit::splitId));
         this.fileSplitStrategy = fileSplitStrategy;
@@ -95,9 +85,10 @@ public class MultipleTableFileSourceSplitEnumerator
     public void open() {
         boolean hasMultiSplits = false;
         Map<String, Integer> splitCountByTable = new HashMap<>();
-        for (Map.Entry<String, List<String>> filePathEntry : filePathMap.entrySet()) {
-            String tableId = filePathEntry.getKey();
-            List<String> filePaths = filePathEntry.getValue();
+        for (BaseFileSourceConfig fileSourceConfig : fileSourceConfigs) {
+            String tableId =
+                    fileSourceConfig.getCatalogTable().getTableId().toTablePath().toString();
+            List<String> filePaths = fileSourceConfig.getFilePathsForSplitEnumerator();
             for (String filePath : filePaths) {
                 List<FileSourceSplit> splits = fileSplitStrategy.split(tableId, filePath);
                 splitCountByTable.merge(tableId, splits.size(), Integer::sum);
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/MultipleTableFileSourceSplitEnumeratorTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/MultipleTableFileSourceSplitEnumeratorTest.java
index 8e4a35f704..6028fb48a8 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/MultipleTableFileSourceSplitEnumeratorTest.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/MultipleTableFileSourceSplitEnumeratorTest.java
@@ -20,42 +20,57 @@ package org.apache.seatunnel.connectors.seatunnel.file.source.split;
 import org.apache.seatunnel.shade.com.google.common.collect.Lists;
 import org.apache.seatunnel.shade.com.google.common.collect.Maps;
 
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
 import org.apache.seatunnel.api.source.SourceSplitEnumerator;
 import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
 import org.apache.seatunnel.api.table.catalog.TableIdentifier;
 import org.apache.seatunnel.connectors.seatunnel.file.config.BaseFileSourceConfig;
 import org.apache.seatunnel.connectors.seatunnel.file.config.BaseMultipleTableFileSourceConfig;
+import org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSourceOptions;
+import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf;
+import org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Progressable;
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
-import lombok.extern.slf4j.Slf4j;
-
+import java.io.IOException;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.IntStream;
 
-@Slf4j
-public class MultipleTableFileSourceSplitEnumeratorTest {
+class MultipleTableFileSourceSplitEnumeratorTest {
+
+    private static final String COUNTING_FILE_SYSTEM_URI = "counting:///";
+    private static final String SOURCE_PATH = "counting:///source";
 
     @Test
     void assignSplitTest() throws Exception {
         int parallelism = 4;
         int fileSize = 50;
 
-        Map<String, List<String>> filePathMap = new HashMap<>();
         List<String> filePaths = new ArrayList<>();
         IntStream.range(0, fileSize).forEach(i -> filePaths.add("filePath" + i));
-        filePathMap.put("table1", filePaths);
 
         BaseFileSourceConfig baseFileSourceConfig = Mockito.mock(BaseFileSourceConfig.class);
 
-        Mockito.when(baseFileSourceConfig.getFilePaths()).thenReturn(filePaths);
+        Mockito.when(baseFileSourceConfig.getFilePathsForSplitEnumerator()).thenReturn(filePaths);
 
         CatalogTable catalogTable =
                 CatalogTable.of(
@@ -100,26 +115,279 @@ public class MultipleTableFileSourceSplitEnumeratorTest {
                     allocateFiles(i, parallelism, fileSize), splitAllValues.get(i).size());
         }
 
-        // check no duplicate file assigned
         Assertions.assertEquals(0, enumerator.currentUnassignedSplitSize());
     }
 
-    /**
-     * calculate the number of files assigned each time
-     *
-     * @param id id
-     * @param parallelism parallelism
-     * @param fileSize file size
-     * @return
-     */
-    public int allocateFiles(int id, int parallelism, int fileSize) {
+    @Test
+    void deferredFileDiscoveryDoesNotListFilesDuringConfigCreation() throws IOException {
+        CountingFileSystem.reset();
+
+        BaseFileSourceConfig sourceConfig =
+                new DeferredFileSourceConfig(
+                        createBinaryConfig(), CatalogTableUtil.buildSimpleTextTable());
+
+        Assertions.assertTrue(sourceConfig.isFileDiscoveryDeferred());
+        Assertions.assertTrue(sourceConfig.getFilePaths().isEmpty());
+        Assertions.assertEquals(0, CountingFileSystem.listStatusCount);
+        Assertions.assertEquals(0, CountingFileSystem.closeCount);
+
+        List<String> filePaths = sourceConfig.getFilePathsForSplitEnumerator();
+
+        Assertions.assertEquals(2, filePaths.size());
+        Assertions.assertEquals(1, CountingFileSystem.listStatusCount);
+        Assertions.assertEquals(1, CountingFileSystem.closeCount);
+
+        Assertions.assertEquals(
+                2, sourceConfig.getReadStrategy().getFileNamesByPath(SOURCE_PATH).size());
+        Assertions.assertEquals(2, CountingFileSystem.listStatusCount);
+        Assertions.assertEquals(1, CountingFileSystem.closeCount);
+    }
+
+    @Test
+    void deferredFileDiscoveryClosesResourcesWhenDiscoveryFails() {
+        CountingFileSystem.reset();
+        CountingFileSystem.failOnListStatus = true;
+        BaseFileSourceConfig sourceConfig =
+                new DeferredFileSourceConfig(
+                        createBinaryConfig(), CatalogTableUtil.buildSimpleTextTable());
+
+        Assertions.assertThrows(
+                FileConnectorException.class, sourceConfig::getFilePathsForSplitEnumerator);
+        Assertions.assertEquals(1, CountingFileSystem.closeCount);
+    }
+
+    @Test
+    void eagerFileDiscoveryKeepsExistingConfigCreationBehavior() {
+        CountingFileSystem.reset();
+
+        BaseFileSourceConfig sourceConfig =
+                new EagerFileSourceConfig(
+                        createBinaryConfig(), CatalogTableUtil.buildSimpleTextTable());
+
+        Assertions.assertFalse(sourceConfig.isFileDiscoveryDeferred());
+        Assertions.assertEquals(2, sourceConfig.getFilePaths().size());
+        Assertions.assertEquals(1, CountingFileSystem.listStatusCount);
+
+        sourceConfig.getFilePathsForSplitEnumerator();
+
+        Assertions.assertEquals(1, CountingFileSystem.listStatusCount);
+    }
+
+    @Test
+    void splitEnumeratorDiscoversDeferredFilePathsOnOpen() {
+        CountingFileSystem.reset();
+        BaseMultipleTableFileSourceConfig multipleTableConfig =
+                createMultipleTableConfig(DeferredFileSourceConfig::new);
+
+        @SuppressWarnings("unchecked")
+        SourceSplitEnumerator.Context<FileSourceSplit> context =
+                Mockito.mock(SourceSplitEnumerator.Context.class);
+
+        MultipleTableFileSourceSplitEnumerator enumerator =
+                new MultipleTableFileSourceSplitEnumerator(
+                        context, multipleTableConfig, new DefaultFileSplitStrategy());
+
+        Assertions.assertEquals(0, CountingFileSystem.listStatusCount);
+        Assertions.assertEquals(0, CountingFileSystem.closeCount);
+
+        enumerator.open();
+
+        Assertions.assertEquals(1, CountingFileSystem.listStatusCount);
+        Assertions.assertEquals(1, CountingFileSystem.closeCount);
+        Assertions.assertEquals(2, enumerator.currentUnassignedSplitSize());
+    }
+
+    private int allocateFiles(int id, int parallelism, int fileSize) {
         int filesPerIteration = fileSize / parallelism;
         int remainder = fileSize % parallelism;
 
         if (id < remainder) {
             return filesPerIteration + 1;
-        } else {
-            return filesPerIteration;
+        }
+        return filesPerIteration;
+    }
+
+    private static ReadonlyConfig createBinaryConfig() {
+        Map<String, Object> config = new HashMap<>();
+        config.put(FileBaseSourceOptions.FILE_PATH.key(), SOURCE_PATH);
+        config.put(FileBaseSourceOptions.FILE_FORMAT_TYPE.key(), "binary");
+        return ReadonlyConfig.fromMap(config);
+    }
+
+    private static BaseMultipleTableFileSourceConfig createMultipleTableConfig(
+            FileSourceConfigFactory fileSourceConfigFactory) {
+        return new BaseMultipleTableFileSourceConfig(
+                createBinaryConfig(),
+                Collections.singletonList(CatalogTableUtil.buildSimpleTextTable())) {
+
+            @Override
+            public BaseFileSourceConfig getBaseSourceConfig(
+                    ReadonlyConfig readonlyConfig, CatalogTable catalogTableFromConfig) {
+                return fileSourceConfigFactory.create(readonlyConfig, catalogTableFromConfig);
+            }
+        };
+    }
+
+    private interface FileSourceConfigFactory {
+
+        BaseFileSourceConfig create(
+                ReadonlyConfig readonlyConfig, CatalogTable catalogTableFromConfig);
+    }
+
+    private static class DeferredFileSourceConfig extends EagerFileSourceConfig {
+
+        private DeferredFileSourceConfig(
+                ReadonlyConfig readonlyConfig, CatalogTable catalogTableFromConfig) {
+            super(readonlyConfig, catalogTableFromConfig);
+        }
+
+        @Override
+        protected boolean shouldDeferFileDiscovery(ReadonlyConfig readonlyConfig) {
+            return true;
+        }
+    }
+
+    private static class EagerFileSourceConfig extends BaseFileSourceConfig {
+
+        private static final long serialVersionUID = 1L;
+
+        private EagerFileSourceConfig(
+                ReadonlyConfig readonlyConfig, CatalogTable catalogTableFromConfig) {
+            super(readonlyConfig, catalogTableFromConfig);
+        }
+
+        @Override
+        public HadoopConf getHadoopConfig() {
+            return new CountingConf();
+        }
+
+        @Override
+        public String getPluginName() {
+            return "CountingFile";
+        }
+    }
+
+    private static class CountingConf extends HadoopConf {
+
+        private static final String SCHEMA = "counting";
+
+        private CountingConf() {
+            super(COUNTING_FILE_SYSTEM_URI);
+        }
+
+        @Override
+        public String getFsHdfsImpl() {
+            return CountingFileSystem.class.getName();
+        }
+
+        @Override
+        public String getSchema() {
+            return SCHEMA;
+        }
+    }
+
+    public static class CountingFileSystem extends FileSystem {
+
+        private static int listStatusCount;
+        private static int closeCount;
+        private static boolean failOnListStatus;
+
+        private URI uri;
+        private Path workingDirectory;
+        private boolean closed;
+
+        private static void reset() {
+            listStatusCount = 0;
+            closeCount = 0;
+            failOnListStatus = false;
+        }
+
+        @Override
+        public void initialize(URI name, Configuration conf) throws IOException {
+            super.initialize(name, conf);
+            this.uri = name;
+            this.workingDirectory = new Path("/");
+        }
+
+        @Override
+        public URI getUri() {
+            return uri;
+        }
+
+        @Override
+        public FSDataInputStream open(Path path, int bufferSize) {
+            throw new UnsupportedOperationException("open is not needed for this test");
+        }
+
+        @Override
+        public FSDataOutputStream create(
+                Path path,
+                FsPermission permission,
+                boolean overwrite,
+                int bufferSize,
+                short replication,
+                long blockSize,
+                Progressable progress) {
+            throw new UnsupportedOperationException("create is not needed for this test");
+        }
+
+        @Override
+        public FSDataOutputStream append(Path path, int bufferSize, Progressable progress) {
+            throw new UnsupportedOperationException("append is not needed for this test");
+        }
+
+        @Override
+        public boolean rename(Path source, Path target) {
+            throw new UnsupportedOperationException("rename is not needed for this test");
+        }
+
+        @Override
+        public boolean delete(Path path, boolean recursive) {
+            throw new UnsupportedOperationException("delete is not needed for this test");
+        }
+
+        @Override
+        public FileStatus[] listStatus(Path path) throws IOException {
+            if (closed) {
+                throw new IOException("File system is closed");
+            }
+            listStatusCount++;
+            if (failOnListStatus) {
+                throw new IOException("File discovery failed");
+            }
+            return new FileStatus[] {fileStatus(path, "first.bin"), fileStatus(path, "second.bin")};
+        }
+
+        @Override
+        public void setWorkingDirectory(Path newDirectory) {
+            this.workingDirectory = newDirectory;
+        }
+
+        @Override
+        public Path getWorkingDirectory() {
+            return workingDirectory;
+        }
+
+        @Override
+        public boolean mkdirs(Path path, FsPermission permission) {
+            return true;
+        }
+
+        @Override
+        public FileStatus getFileStatus(Path path) {
+            return new FileStatus(0, true, 1, 0, 0, path.makeQualified(uri, workingDirectory));
+        }
+
+        @Override
+        public void close() throws IOException {
+            closed = true;
+            closeCount++;
+            super.close();
+        }
+
+        private FileStatus fileStatus(Path parent, String fileName) {
+            Path path = new Path(parent, fileName).makeQualified(uri, workingDirectory);
+            return new FileStatus(1, false, 1, 0, 0, path);
         }
     }
 }
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/config/SFTPFileSourceConfig.java b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/config/SFTPFileSourceConfig.java
index c4a01f475e..e823a5afc0 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/config/SFTPFileSourceConfig.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/config/SFTPFileSourceConfig.java
@@ -20,6 +20,9 @@ package org.apache.seatunnel.connectors.seatunnel.file.sftp.config;
 import org.apache.seatunnel.api.configuration.ReadonlyConfig;
 import org.apache.seatunnel.api.table.catalog.CatalogTable;
 import org.apache.seatunnel.connectors.seatunnel.file.config.BaseFileSourceConfig;
+import org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSourceOptions;
+import org.apache.seatunnel.connectors.seatunnel.file.config.FileDiscoveryMode;
+import org.apache.seatunnel.connectors.seatunnel.file.config.FileFormat;
 import org.apache.seatunnel.connectors.seatunnel.file.config.FileSystemType;
 import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf;
 
@@ -40,6 +43,13 @@ public class SFTPFileSourceConfig extends BaseFileSourceConfig {
         return FileSystemType.SFTP.getFileSystemPluginName();
     }
 
+    @Override
+    protected boolean shouldDeferFileDiscovery(ReadonlyConfig readonlyConfig) {
+        return readonlyConfig.get(FileBaseSourceOptions.FILE_FORMAT_TYPE) == FileFormat.BINARY
+                && readonlyConfig.get(FileBaseSourceOptions.DISCOVERY_MODE)
+                        == FileDiscoveryMode.ONCE;
+    }
+
     public SFTPFileSourceConfig(
             ReadonlyConfig readonlyConfig, CatalogTable catalogTableFromConfig) {
         super(readonlyConfig, catalogTableFromConfig);