You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by li...@apache.org on 2026/07/21 14:28:40 UTC
(seatunnel) branch dev updated: [Improve][Connector-V2][SftpFile] Defer binary file discovery (#11377)
This is an automated email from the ASF dual-hosted git repository.
davidzollo pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 886c0fd13a [Improve][Connector-V2][SftpFile] Defer binary file discovery (#11377)
886c0fd13a is described below
commit 886c0fd13ad7e7e8a6e782299e7349dfa814ed7e
Author: Goutam Adwant <86...@users.noreply.github.com>
AuthorDate: Tue Jul 21 16:28:33 2026 +0200
[Improve][Connector-V2][SftpFile] Defer binary file discovery (#11377)
Signed-off-by: goutamadwant <wo...@gmail.com>
---
.../file/config/BaseFileSourceConfig.java | 78 +++++-
.../MultipleTableFileSourceSplitEnumerator.java | 21 +-
...MultipleTableFileSourceSplitEnumeratorTest.java | 306 +++++++++++++++++++--
.../file/sftp/config/SFTPFileSourceConfig.java | 10 +
4 files changed, 376 insertions(+), 39 deletions(-)
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseFileSourceConfig.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseFileSourceConfig.java
index 604cf90db3..08a0902f44 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseFileSourceConfig.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseFileSourceConfig.java
@@ -33,8 +33,11 @@ import org.apache.seatunnel.connectors.seatunnel.file.source.reader.ReadStrategy
import org.apache.commons.collections4.CollectionUtils;
import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import java.io.IOException;
import java.io.Serializable;
+import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -43,6 +46,7 @@ import java.util.function.Function;
import java.util.stream.Collectors;
@Getter
+@Slf4j
public abstract class BaseFileSourceConfig implements Serializable {
private static final long serialVersionUID = 1L;
@@ -53,6 +57,7 @@ public abstract class BaseFileSourceConfig implements Serializable {
private final List<String> filePaths;
private final ReadonlyConfig baseFileSourceConfig;
private final CatalogTable catalogTableFromConfig;
+ private final boolean fileDiscoveryDeferred;
public abstract HadoopConf getHadoopConfig();
@@ -63,20 +68,61 @@ public abstract class BaseFileSourceConfig implements Serializable {
this.baseFileSourceConfig = readonlyConfig;
this.fileFormat = readonlyConfig.get(FileBaseSourceOptions.FILE_FORMAT_TYPE);
this.readStrategy = ReadStrategyFactory.of(readonlyConfig, getHadoopConfig());
+ this.fileDiscoveryDeferred = shouldDeferFileDiscovery(readonlyConfig);
this.filePaths = parseFilePaths(readonlyConfig);
this.catalogTableFromConfig = catalogTableFromConfig;
this.catalogTable = parseCatalogTable(readonlyConfig);
}
+ protected boolean shouldDeferFileDiscovery(ReadonlyConfig readonlyConfig) {
+ return false;
+ }
+
private List<String> parseFilePaths(ReadonlyConfig readonlyConfig) {
- if (readonlyConfig.get(FileBaseSourceOptions.DISCOVERY_MODE)
- == FileDiscoveryMode.CONTINUOUS) {
+ if (readonlyConfig.get(FileBaseSourceOptions.DISCOVERY_MODE) == FileDiscoveryMode.CONTINUOUS
+ || fileDiscoveryDeferred) {
return Collections.emptyList();
}
- String rootPath = null;
+ return discoverFilePaths();
+ }
+
+ public List<String> getFilePathsForSplitEnumerator() {
+ if (fileDiscoveryDeferred) {
+ ReadStrategy discoveryReadStrategy =
+ ReadStrategyFactory.of(baseFileSourceConfig, getHadoopConfig());
+ try {
+ return discoverFilePaths(discoveryReadStrategy);
+ } finally {
+ closeDiscoveryReadStrategy(discoveryReadStrategy);
+ }
+ }
+ return filePaths;
+ }
+
+ private void closeDiscoveryReadStrategy(ReadStrategy discoveryReadStrategy) {
+ try {
+ discoveryReadStrategy.close();
+ } catch (IOException e) {
+ log.warn("Failed to close file discovery resources for plugin {}", getPluginName(), e);
+ }
+ }
+
+ private List<String> discoverFilePaths() {
+ return discoverFilePaths(readStrategy);
+ }
+
+ private List<String> discoverFilePaths(ReadStrategy discoveryReadStrategy) {
+ String rootPath = baseFileSourceConfig.get(FileBaseSourceOptions.FILE_PATH);
+ long startTime = System.currentTimeMillis();
try {
- rootPath = readonlyConfig.get(FileBaseSourceOptions.FILE_PATH);
- return readStrategy.getFileNamesByPath(rootPath);
+ List<String> discoveredFilePaths = discoveryReadStrategy.getFileNamesByPath(rootPath);
+ log.info(
+ "File source discovery finished: plugin={}, path={}, files={}, cost={}ms",
+ getPluginName(),
+ maskUriUserInfo(rootPath),
+ discoveredFilePaths.size(),
+ System.currentTimeMillis() - startTime);
+ return discoveredFilePaths;
} catch (Exception ex) {
String errorMsg = String.format("Get file list from this path [%s] failed", rootPath);
throw new FileConnectorException(
@@ -127,6 +173,28 @@ public abstract class BaseFileSourceConfig implements Serializable {
return readStrategy.getSeaTunnelRowTypeInfo(rootPath);
}
+ private static String maskUriUserInfo(String rawPath) {
+ if (rawPath == null) {
+ return null;
+ }
+ try {
+ URI uri = URI.create(rawPath);
+ if (uri.getUserInfo() == null || uri.getAuthority() == null) {
+ return rawPath;
+ }
+ String maskedAuthority = uri.getAuthority().replace(uri.getUserInfo() + "@", "***@");
+ return new URI(
+ uri.getScheme(),
+ maskedAuthority,
+ uri.getPath(),
+ uri.getQuery(),
+ uri.getFragment())
+ .toString();
+ } catch (Exception e) {
+ return rawPath;
+ }
+ }
+
private CatalogTable newCatalogTable(
CatalogTable catalogTable, SeaTunnelRowType seaTunnelRowType) {
TableSchema tableSchema = catalogTable.getTableSchema();
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/MultipleTableFileSourceSplitEnumerator.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/MultipleTableFileSourceSplitEnumerator.java
index b0c5dd2ad9..92acf52c80 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/MultipleTableFileSourceSplitEnumerator.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/MultipleTableFileSourceSplitEnumerator.java
@@ -48,7 +48,7 @@ public class MultipleTableFileSourceSplitEnumerator
private final Context<FileSourceSplit> context;
private final Set<FileSourceSplit> allSplit;
private final Set<FileSourceSplit> assignedSplit;
- private final Map<String, List<String>> filePathMap;
+ private final List<BaseFileSourceConfig> fileSourceConfigs;
private final AtomicInteger assignCount = new AtomicInteger(0);
private final Object lock = new Object();
private final FileSplitStrategy fileSplitStrategy;
@@ -58,17 +58,7 @@ public class MultipleTableFileSourceSplitEnumerator
BaseMultipleTableFileSourceConfig multipleTableFileSourceConfig,
FileSplitStrategy fileSplitStrategy) {
this.context = context;
- this.filePathMap =
- multipleTableFileSourceConfig.getFileSourceConfigs().stream()
- .collect(
- Collectors.toMap(
- localFileSourceConfig ->
- localFileSourceConfig
- .getCatalogTable()
- .getTableId()
- .toTablePath()
- .toString(),
- BaseFileSourceConfig::getFilePaths));
+ this.fileSourceConfigs = multipleTableFileSourceConfig.getFileSourceConfigs();
this.assignedSplit = new HashSet<>();
this.allSplit = new TreeSet<>(Comparator.comparing(FileSourceSplit::splitId));
this.fileSplitStrategy = fileSplitStrategy;
@@ -95,9 +85,10 @@ public class MultipleTableFileSourceSplitEnumerator
public void open() {
boolean hasMultiSplits = false;
Map<String, Integer> splitCountByTable = new HashMap<>();
- for (Map.Entry<String, List<String>> filePathEntry : filePathMap.entrySet()) {
- String tableId = filePathEntry.getKey();
- List<String> filePaths = filePathEntry.getValue();
+ for (BaseFileSourceConfig fileSourceConfig : fileSourceConfigs) {
+ String tableId =
+ fileSourceConfig.getCatalogTable().getTableId().toTablePath().toString();
+ List<String> filePaths = fileSourceConfig.getFilePathsForSplitEnumerator();
for (String filePath : filePaths) {
List<FileSourceSplit> splits = fileSplitStrategy.split(tableId, filePath);
splitCountByTable.merge(tableId, splits.size(), Integer::sum);
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/MultipleTableFileSourceSplitEnumeratorTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/MultipleTableFileSourceSplitEnumeratorTest.java
index 8e4a35f704..6028fb48a8 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/MultipleTableFileSourceSplitEnumeratorTest.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/MultipleTableFileSourceSplitEnumeratorTest.java
@@ -20,42 +20,57 @@ package org.apache.seatunnel.connectors.seatunnel.file.source.split;
import org.apache.seatunnel.shade.com.google.common.collect.Lists;
import org.apache.seatunnel.shade.com.google.common.collect.Maps;
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
import org.apache.seatunnel.api.source.SourceSplitEnumerator;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
import org.apache.seatunnel.api.table.catalog.TableIdentifier;
import org.apache.seatunnel.connectors.seatunnel.file.config.BaseFileSourceConfig;
import org.apache.seatunnel.connectors.seatunnel.file.config.BaseMultipleTableFileSourceConfig;
+import org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSourceOptions;
+import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf;
+import org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Progressable;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
-import lombok.extern.slf4j.Slf4j;
-
+import java.io.IOException;
+import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.IntStream;
-@Slf4j
-public class MultipleTableFileSourceSplitEnumeratorTest {
+class MultipleTableFileSourceSplitEnumeratorTest {
+
+ private static final String COUNTING_FILE_SYSTEM_URI = "counting:///";
+ private static final String SOURCE_PATH = "counting:///source";
@Test
void assignSplitTest() throws Exception {
int parallelism = 4;
int fileSize = 50;
- Map<String, List<String>> filePathMap = new HashMap<>();
List<String> filePaths = new ArrayList<>();
IntStream.range(0, fileSize).forEach(i -> filePaths.add("filePath" + i));
- filePathMap.put("table1", filePaths);
BaseFileSourceConfig baseFileSourceConfig = Mockito.mock(BaseFileSourceConfig.class);
- Mockito.when(baseFileSourceConfig.getFilePaths()).thenReturn(filePaths);
+ Mockito.when(baseFileSourceConfig.getFilePathsForSplitEnumerator()).thenReturn(filePaths);
CatalogTable catalogTable =
CatalogTable.of(
@@ -100,26 +115,279 @@ public class MultipleTableFileSourceSplitEnumeratorTest {
allocateFiles(i, parallelism, fileSize), splitAllValues.get(i).size());
}
- // check no duplicate file assigned
Assertions.assertEquals(0, enumerator.currentUnassignedSplitSize());
}
- /**
- * calculate the number of files assigned each time
- *
- * @param id id
- * @param parallelism parallelism
- * @param fileSize file size
- * @return
- */
- public int allocateFiles(int id, int parallelism, int fileSize) {
+ @Test
+ void deferredFileDiscoveryDoesNotListFilesDuringConfigCreation() throws IOException {
+ CountingFileSystem.reset();
+
+ BaseFileSourceConfig sourceConfig =
+ new DeferredFileSourceConfig(
+ createBinaryConfig(), CatalogTableUtil.buildSimpleTextTable());
+
+ Assertions.assertTrue(sourceConfig.isFileDiscoveryDeferred());
+ Assertions.assertTrue(sourceConfig.getFilePaths().isEmpty());
+ Assertions.assertEquals(0, CountingFileSystem.listStatusCount);
+ Assertions.assertEquals(0, CountingFileSystem.closeCount);
+
+ List<String> filePaths = sourceConfig.getFilePathsForSplitEnumerator();
+
+ Assertions.assertEquals(2, filePaths.size());
+ Assertions.assertEquals(1, CountingFileSystem.listStatusCount);
+ Assertions.assertEquals(1, CountingFileSystem.closeCount);
+
+ Assertions.assertEquals(
+ 2, sourceConfig.getReadStrategy().getFileNamesByPath(SOURCE_PATH).size());
+ Assertions.assertEquals(2, CountingFileSystem.listStatusCount);
+ Assertions.assertEquals(1, CountingFileSystem.closeCount);
+ }
+
+ @Test
+ void deferredFileDiscoveryClosesResourcesWhenDiscoveryFails() {
+ CountingFileSystem.reset();
+ CountingFileSystem.failOnListStatus = true;
+ BaseFileSourceConfig sourceConfig =
+ new DeferredFileSourceConfig(
+ createBinaryConfig(), CatalogTableUtil.buildSimpleTextTable());
+
+ Assertions.assertThrows(
+ FileConnectorException.class, sourceConfig::getFilePathsForSplitEnumerator);
+ Assertions.assertEquals(1, CountingFileSystem.closeCount);
+ }
+
+ @Test
+ void eagerFileDiscoveryKeepsExistingConfigCreationBehavior() {
+ CountingFileSystem.reset();
+
+ BaseFileSourceConfig sourceConfig =
+ new EagerFileSourceConfig(
+ createBinaryConfig(), CatalogTableUtil.buildSimpleTextTable());
+
+ Assertions.assertFalse(sourceConfig.isFileDiscoveryDeferred());
+ Assertions.assertEquals(2, sourceConfig.getFilePaths().size());
+ Assertions.assertEquals(1, CountingFileSystem.listStatusCount);
+
+ sourceConfig.getFilePathsForSplitEnumerator();
+
+ Assertions.assertEquals(1, CountingFileSystem.listStatusCount);
+ }
+
+ @Test
+ void splitEnumeratorDiscoversDeferredFilePathsOnOpen() {
+ CountingFileSystem.reset();
+ BaseMultipleTableFileSourceConfig multipleTableConfig =
+ createMultipleTableConfig(DeferredFileSourceConfig::new);
+
+ @SuppressWarnings("unchecked")
+ SourceSplitEnumerator.Context<FileSourceSplit> context =
+ Mockito.mock(SourceSplitEnumerator.Context.class);
+
+ MultipleTableFileSourceSplitEnumerator enumerator =
+ new MultipleTableFileSourceSplitEnumerator(
+ context, multipleTableConfig, new DefaultFileSplitStrategy());
+
+ Assertions.assertEquals(0, CountingFileSystem.listStatusCount);
+ Assertions.assertEquals(0, CountingFileSystem.closeCount);
+
+ enumerator.open();
+
+ Assertions.assertEquals(1, CountingFileSystem.listStatusCount);
+ Assertions.assertEquals(1, CountingFileSystem.closeCount);
+ Assertions.assertEquals(2, enumerator.currentUnassignedSplitSize());
+ }
+
+ private int allocateFiles(int id, int parallelism, int fileSize) {
int filesPerIteration = fileSize / parallelism;
int remainder = fileSize % parallelism;
if (id < remainder) {
return filesPerIteration + 1;
- } else {
- return filesPerIteration;
+ }
+ return filesPerIteration;
+ }
+
+ private static ReadonlyConfig createBinaryConfig() {
+ Map<String, Object> config = new HashMap<>();
+ config.put(FileBaseSourceOptions.FILE_PATH.key(), SOURCE_PATH);
+ config.put(FileBaseSourceOptions.FILE_FORMAT_TYPE.key(), "binary");
+ return ReadonlyConfig.fromMap(config);
+ }
+
+ private static BaseMultipleTableFileSourceConfig createMultipleTableConfig(
+ FileSourceConfigFactory fileSourceConfigFactory) {
+ return new BaseMultipleTableFileSourceConfig(
+ createBinaryConfig(),
+ Collections.singletonList(CatalogTableUtil.buildSimpleTextTable())) {
+
+ @Override
+ public BaseFileSourceConfig getBaseSourceConfig(
+ ReadonlyConfig readonlyConfig, CatalogTable catalogTableFromConfig) {
+ return fileSourceConfigFactory.create(readonlyConfig, catalogTableFromConfig);
+ }
+ };
+ }
+
+ private interface FileSourceConfigFactory {
+
+ BaseFileSourceConfig create(
+ ReadonlyConfig readonlyConfig, CatalogTable catalogTableFromConfig);
+ }
+
+ private static class DeferredFileSourceConfig extends EagerFileSourceConfig {
+
+ private DeferredFileSourceConfig(
+ ReadonlyConfig readonlyConfig, CatalogTable catalogTableFromConfig) {
+ super(readonlyConfig, catalogTableFromConfig);
+ }
+
+ @Override
+ protected boolean shouldDeferFileDiscovery(ReadonlyConfig readonlyConfig) {
+ return true;
+ }
+ }
+
+ private static class EagerFileSourceConfig extends BaseFileSourceConfig {
+
+ private static final long serialVersionUID = 1L;
+
+ private EagerFileSourceConfig(
+ ReadonlyConfig readonlyConfig, CatalogTable catalogTableFromConfig) {
+ super(readonlyConfig, catalogTableFromConfig);
+ }
+
+ @Override
+ public HadoopConf getHadoopConfig() {
+ return new CountingConf();
+ }
+
+ @Override
+ public String getPluginName() {
+ return "CountingFile";
+ }
+ }
+
+ private static class CountingConf extends HadoopConf {
+
+ private static final String SCHEMA = "counting";
+
+ private CountingConf() {
+ super(COUNTING_FILE_SYSTEM_URI);
+ }
+
+ @Override
+ public String getFsHdfsImpl() {
+ return CountingFileSystem.class.getName();
+ }
+
+ @Override
+ public String getSchema() {
+ return SCHEMA;
+ }
+ }
+
+ public static class CountingFileSystem extends FileSystem {
+
+ private static int listStatusCount;
+ private static int closeCount;
+ private static boolean failOnListStatus;
+
+ private URI uri;
+ private Path workingDirectory;
+ private boolean closed;
+
+ private static void reset() {
+ listStatusCount = 0;
+ closeCount = 0;
+ failOnListStatus = false;
+ }
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ this.uri = name;
+ this.workingDirectory = new Path("/");
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) {
+ throw new UnsupportedOperationException("open is not needed for this test");
+ }
+
+ @Override
+ public FSDataOutputStream create(
+ Path path,
+ FsPermission permission,
+ boolean overwrite,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) {
+ throw new UnsupportedOperationException("create is not needed for this test");
+ }
+
+ @Override
+ public FSDataOutputStream append(Path path, int bufferSize, Progressable progress) {
+ throw new UnsupportedOperationException("append is not needed for this test");
+ }
+
+ @Override
+ public boolean rename(Path source, Path target) {
+ throw new UnsupportedOperationException("rename is not needed for this test");
+ }
+
+ @Override
+ public boolean delete(Path path, boolean recursive) {
+ throw new UnsupportedOperationException("delete is not needed for this test");
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ if (closed) {
+ throw new IOException("File system is closed");
+ }
+ listStatusCount++;
+ if (failOnListStatus) {
+ throw new IOException("File discovery failed");
+ }
+ return new FileStatus[] {fileStatus(path, "first.bin"), fileStatus(path, "second.bin")};
+ }
+
+ @Override
+ public void setWorkingDirectory(Path newDirectory) {
+ this.workingDirectory = newDirectory;
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDirectory;
+ }
+
+ @Override
+ public boolean mkdirs(Path path, FsPermission permission) {
+ return true;
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path path) {
+ return new FileStatus(0, true, 1, 0, 0, path.makeQualified(uri, workingDirectory));
+ }
+
+ @Override
+ public void close() throws IOException {
+ closed = true;
+ closeCount++;
+ super.close();
+ }
+
+ private FileStatus fileStatus(Path parent, String fileName) {
+ Path path = new Path(parent, fileName).makeQualified(uri, workingDirectory);
+ return new FileStatus(1, false, 1, 0, 0, path);
}
}
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/config/SFTPFileSourceConfig.java b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/config/SFTPFileSourceConfig.java
index c4a01f475e..e823a5afc0 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/config/SFTPFileSourceConfig.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/config/SFTPFileSourceConfig.java
@@ -20,6 +20,9 @@ package org.apache.seatunnel.connectors.seatunnel.file.sftp.config;
import org.apache.seatunnel.api.configuration.ReadonlyConfig;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
import org.apache.seatunnel.connectors.seatunnel.file.config.BaseFileSourceConfig;
+import org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSourceOptions;
+import org.apache.seatunnel.connectors.seatunnel.file.config.FileDiscoveryMode;
+import org.apache.seatunnel.connectors.seatunnel.file.config.FileFormat;
import org.apache.seatunnel.connectors.seatunnel.file.config.FileSystemType;
import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf;
@@ -40,6 +43,13 @@ public class SFTPFileSourceConfig extends BaseFileSourceConfig {
return FileSystemType.SFTP.getFileSystemPluginName();
}
+ @Override
+ protected boolean shouldDeferFileDiscovery(ReadonlyConfig readonlyConfig) {
+ return readonlyConfig.get(FileBaseSourceOptions.FILE_FORMAT_TYPE) == FileFormat.BINARY
+ && readonlyConfig.get(FileBaseSourceOptions.DISCOVERY_MODE)
+ == FileDiscoveryMode.ONCE;
+ }
+
public SFTPFileSourceConfig(
ReadonlyConfig readonlyConfig, CatalogTable catalogTableFromConfig) {
super(readonlyConfig, catalogTableFromConfig);