You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ze...@apache.org on 2026/06/15 01:08:07 UTC
(seatunnel) branch dev updated: [Feature][Connector-V2] Add recursive_file_scan option for file connectors (#10505)
This is an automated email from the ASF dual-hosted git repository.
yzeng1618 pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 2826ea3dcd [Feature][Connector-V2] Add recursive_file_scan option for file connectors (#10505)
2826ea3dcd is described below
commit 2826ea3dcd90350ddcb5391298f1bc52e4448a5d
Author: Kang Myeong Gwan <lu...@gmail.com>
AuthorDate: Mon Jun 15 10:08:01 2026 +0900
[Feature][Connector-V2] Add recursive_file_scan option for file connectors (#10505)
Co-authored-by: 강명관 <mg...@douzone.com>
---
docs/en/connectors/source/CosFile.md | 7 +-
docs/en/connectors/source/FtpFile.md | 8 +-
docs/en/connectors/source/HdfsFile.md | 7 +
docs/en/connectors/source/LocalFile.md | 9 +-
docs/en/connectors/source/ObsFile.md | 2 +-
docs/en/connectors/source/OssFile.md | 7 +
docs/en/connectors/source/OssJindoFile.md | 6 +
docs/en/connectors/source/S3File.md | 7 +
docs/en/connectors/source/SftpFile.md | 7 +
docs/zh/connectors/source/CosFile.md | 7 +-
docs/zh/connectors/source/FtpFile.md | 8 +-
docs/zh/connectors/source/HdfsFile.md | 7 +
docs/zh/connectors/source/LocalFile.md | 8 +-
docs/zh/connectors/source/ObsFile.md | 3 +-
docs/zh/connectors/source/OssFile.md | 8 +-
docs/zh/connectors/source/OssJindoFile.md | 3 +-
docs/zh/connectors/source/S3File.md | 8 +-
docs/zh/connectors/source/SftpFile.md | 8 +-
.../file/config/FileBaseSourceOptions.java | 8 +
.../file/source/reader/AbstractReadStrategy.java | 9 +-
...uousMultipleTableFileSourceSplitEnumerator.java | 10 +-
.../source/reader/AbstractReadStrategyTest.java | 168 +++++++++++++++++++++
.../file/source/reader/UpdateSyncModeTest.java | 77 ++++++++++
...MultipleTableFileSourceSplitEnumeratorTest.java | 114 +++++++++++++-
.../file/cos/source/CosFileSourceFactory.java | 1 +
.../file/ftp/source/FtpFileSourceFactory.java | 1 +
.../file/hdfs/source/HdfsFileSourceFactory.java | 2 +
.../oss/jindo/source/OssFileSourceFactory.java | 1 +
.../file/local/source/LocalFileSourceFactory.java | 1 +
.../file/obs/source/ObsFileSourceFactory.java | 1 +
.../file/oss/source/OssFileSourceFactory.java | 1 +
.../file/s3/source/S3FileSourceFactory.java | 1 +
.../file/sftp/source/SftpFileSourceFactory.java | 1 +
.../e2e/connector/file/ftp/FtpFileIT.java | 127 +++++++++++++++-
...ary_update_distcp_continuous_non_recursive.conf | 57 +++++++
.../ftp_binary_update_non_recursive_distcp.conf | 52 +++++++
...inary_update_non_recursive_strict_checksum.conf | 52 +++++++
.../ftp_file_text_non_recursive_to_assert.conf | 121 +++++++++++++++
.../text/ftp_file_text_recursive_to_assert.conf | 121 +++++++++++++++
.../e2e/connector/file/hdfs/HdfsFileIT.java | 139 +++++++++++++++++
...ary_update_distcp_continuous_non_recursive.conf | 58 +++++++
.../hdfs_binary_update_non_recursive_distcp.conf | 53 +++++++
...inary_update_non_recursive_strict_checksum.conf | 53 +++++++
.../hdfs_text_non_recursive_to_assert.conf | 113 ++++++++++++++
.../resources/hdfs_text_recursive_to_assert.conf | 113 ++++++++++++++
.../src/test/resources/text/e2e.txt | 5 +
.../e2e/connector/file/local/LocalFileIT.java | 128 ++++++++++++++++
...ary_update_distcp_continuous_non_recursive.conf | 47 ++++++
...al_file_binary_update_non_recursive_distcp.conf | 42 ++++++
...inary_update_non_recursive_strict_checksum.conf | 42 ++++++
.../local_file_text_non_recursive_to_assert.conf | 117 ++++++++++++++
.../text/local_file_text_recursive_to_assert.conf | 117 ++++++++++++++
.../e2e/connector/file/fstp/SftpFileIT.java | 137 +++++++++++++++++
...ary_update_distcp_continuous_non_recursive.conf | 64 ++++++++
.../sftp_binary_update_non_recursive_distcp.conf | 59 ++++++++
...inary_update_non_recursive_strict_checksum.conf | 59 ++++++++
.../sftp_file_text_non_recursive_to_assert.conf | 122 +++++++++++++++
.../text/sftp_file_text_recursive_to_assert.conf | 122 +++++++++++++++
58 files changed, 2609 insertions(+), 27 deletions(-)
diff --git a/docs/en/connectors/source/CosFile.md b/docs/en/connectors/source/CosFile.md
index 4c71220b22..d6a793f8cb 100644
--- a/docs/en/connectors/source/CosFile.md
+++ b/docs/en/connectors/source/CosFile.md
@@ -85,6 +85,7 @@ To use this connector you need put hadoop-cos-{hadoop.version}-{version}.jar and
| file_filter_modified_end | string | no | - |
| quote_char | string | no | " |
| escape_char | string | no | - |
+| recursive_file_scan | boolean | no | true |
| sort_files_by_modification_time | boolean | no | false |
### path [string]
@@ -437,6 +438,11 @@ A single character that encloses CSV fields, allowing fields with commas, line b
A single character that allows the quote or other special characters to appear inside a CSV field without ending the field.
+### recursive_file_scan [boolean]
+
+Whether to scan subdirectories recursively.
+If `false`, subdirectories will be ignored.
+
### sort_files_by_modification_time [boolean]
Whether to sort files by modification time in descending order. Default is `false`.
@@ -548,4 +554,3 @@ sink {
## Changelog
<ChangeLog />
-
diff --git a/docs/en/connectors/source/FtpFile.md b/docs/en/connectors/source/FtpFile.md
index a04def1680..87589f5945 100644
--- a/docs/en/connectors/source/FtpFile.md
+++ b/docs/en/connectors/source/FtpFile.md
@@ -89,8 +89,9 @@ If you use SeaTunnel Engine, It automatically integrated the hadoop jar when you
| file_filter_modified_end | string | no | - |
| quote_char | string | no | " |
| escape_char | string | no | - |
-| sort_files_by_modification_time | boolean | no | false |
| metalake_type | string | no | gravitino |
+| recursive_file_scan | boolean | no | true |
+| sort_files_by_modification_time | boolean | no | false |
### host [string]
@@ -539,6 +540,11 @@ A single character that encloses CSV fields, allowing fields with commas, line b
A single character that allows the quote or other special characters to appear inside a CSV field without ending the field.
+### recursive_file_scan [boolean]
+
+Whether to scan subdirectories recursively.
+If `false`, subdirectories will be ignored.
+
### sort_files_by_modification_time [boolean]
Whether to sort files by modification time in descending order. Default is `false`.
diff --git a/docs/en/connectors/source/HdfsFile.md b/docs/en/connectors/source/HdfsFile.md
index 8124c1a6c6..514f1485de 100644
--- a/docs/en/connectors/source/HdfsFile.md
+++ b/docs/en/connectors/source/HdfsFile.md
@@ -95,6 +95,8 @@ Read data from hdfs file system.
| file_split_size | long | no | 134217728 | Split size in bytes when `enable_file_split=true`. For `text`/`csv`/`json`, the split end will be aligned to the next `row_delimiter`. For `parquet`, the split unit is RowGroup and will never break a RowGroup. |
| quote_char | string | no | " | A single character that encloses CSV fields, allowing fields with commas, line breaks, or quotes to be read correctly. |
| escape_char | string | no | - | A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. |
+| metalake_type | string | no | gravitino | The type of metalake service, currently supports `gravitino`. |
+| recursive_file_scan | boolean | no | true | Whether to scan subdirectories recursively. If `false`, subdirectories will be ignored. |
| sort_files_by_modification_time | boolean | no | false | Sort files by modification time in descending order. Enable this when reading evolving schemas to ensure schema inference uses the latest file. |
### file_format_type [string]
@@ -321,6 +323,11 @@ A single character that encloses CSV fields, allowing fields with commas, line b
A single character that allows the quote or other special characters to appear inside a CSV field without ending the field.
+### recursive_file_scan [boolean]
+
+Whether to scan subdirectories recursively.
+If `false`, subdirectories will be ignored.
+
### sort_files_by_modification_time [boolean]
Whether to sort files by modification time in descending order. Default is `false`.
diff --git a/docs/en/connectors/source/LocalFile.md b/docs/en/connectors/source/LocalFile.md
index 44b208b8ac..de5f1c43d7 100644
--- a/docs/en/connectors/source/LocalFile.md
+++ b/docs/en/connectors/source/LocalFile.md
@@ -92,7 +92,9 @@ If you use SeaTunnel Engine, It automatically integrated the hadoop jar when you
| file_split_size | long | no | 134217728 |
| quote_char | string | no | " |
| escape_char | string | no | - |
-| sort_files_by_modification_time | boolean | no | false | Sort files by modification time in descending order. Enable this when reading evolving schemas to ensure schema inference uses the latest file. |
+| metalake_type | string | no | gravitino |
+| recursive_file_scan | boolean | no | true |
+| sort_files_by_modification_time | boolean | no | false |
### path [string]
The source file path.
@@ -535,6 +537,11 @@ A single character that encloses CSV fields, allowing fields with commas, line b
A single character that allows the quote or other special characters to appear inside a CSV field without ending the field.
+### recursive_file_scan [boolean]
+
+Whether to scan subdirectories recursively.
+If `false`, subdirectories will be ignored.
+
### sort_files_by_modification_time [boolean]
Whether to sort files by modification time in descending order. Default is `false`.
diff --git a/docs/en/connectors/source/ObsFile.md b/docs/en/connectors/source/ObsFile.md
index 4ecc0d5a21..618885a346 100644
--- a/docs/en/connectors/source/ObsFile.md
+++ b/docs/en/connectors/source/ObsFile.md
@@ -86,6 +86,7 @@ It only supports hadoop version **2.9.X+**.
| file_filter_modified_end | string | no | - | File modification time filter. The connector will filter some files base on the last modification end time (not include end time). The default data format is `yyyy-MM-dd HH:mm:ss`. |
| quote_char | string | no | " | A single character that encloses CSV fields, allowing fields with commas, line breaks, or quotes to be read correctly. |
| escape_char | string | no | - | A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. |
+| recursive_file_scan | boolean | no | true | Whether to scan subdirectories recursively. If `false`, subdirectories will be ignored. |
| sort_files_by_modification_time | boolean | no | false | Sort files by modification time in descending order. Enable this when reading evolving schemas to ensure schema inference uses the latest file. |
### Tips
@@ -390,4 +391,3 @@ schema {
## Changelog
<ChangeLog />
-
diff --git a/docs/en/connectors/source/OssFile.md b/docs/en/connectors/source/OssFile.md
index 8636bce177..6bc5fb63d5 100644
--- a/docs/en/connectors/source/OssFile.md
+++ b/docs/en/connectors/source/OssFile.md
@@ -217,6 +217,8 @@ If you assign file type to `parquet` `orc`, schema option not required, connecto
| file_filter_modified_end | string | no | - | File modification time filter. The connector will filter some files base on the last modification end time (not include end time). The default data format is `yyyy-MM-dd HH:mm:ss`. |
| quote_char | string | no | " | A single character that encloses CSV fields, allowing fields with commas, line breaks, or quotes to be read correctly. |
| escape_char | string | no | - | A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. |
+| metalake_type | string | no | gravitino | The type of metalake service, currently supports `gravitino`. |
+| recursive_file_scan | boolean | no | true | Whether to scan subdirectories recursively. If `false`, subdirectories will be ignored. |
| sort_files_by_modification_time | boolean | no | false | Sort files by modification time in descending order. Enable this when reading evolving schemas to ensure schema inference uses the latest file. |
### file_format_type [string]
@@ -283,6 +285,11 @@ A single character that encloses CSV fields, allowing fields with commas, line b
A single character that allows the quote or other special characters to appear inside a CSV field without ending the field.
+### recursive_file_scan [boolean]
+
+Whether to scan subdirectories recursively.
+If `false`, subdirectories will be ignored.
+
### sort_files_by_modification_time [boolean]
Whether to sort files by modification time in descending order. Default is `false`.
diff --git a/docs/en/connectors/source/OssJindoFile.md b/docs/en/connectors/source/OssJindoFile.md
index 6cadda858c..7a4f965480 100644
--- a/docs/en/connectors/source/OssJindoFile.md
+++ b/docs/en/connectors/source/OssJindoFile.md
@@ -87,6 +87,7 @@ It only supports hadoop version **2.9.X+**.
| file_filter_modified_end | string | no | - |
| quote_char | string | no | " |
| escape_char | string | no | - |
+| recursive_file_scan | boolean | no | true |
| sort_files_by_modification_time | boolean | no | false |
### path [string]
@@ -418,6 +419,11 @@ A single character that encloses CSV fields, allowing fields with commas, line b
A single character that allows the quote or other special characters to appear inside a CSV field without ending the field.
+### recursive_file_scan [boolean]
+
+Whether to scan subdirectories recursively.
+If `false`, subdirectories will be ignored.
+
### sort_files_by_modification_time [boolean]
Whether to sort files by modification time in descending order. Default is `false`.
diff --git a/docs/en/connectors/source/S3File.md b/docs/en/connectors/source/S3File.md
index b4ccfda5e9..71bd4cedd9 100644
--- a/docs/en/connectors/source/S3File.md
+++ b/docs/en/connectors/source/S3File.md
@@ -226,6 +226,8 @@ If you assign file type to `parquet` `orc`, schema option not required, connecto
| common-options | | no | - | Source plugin common parameters, please refer to [Source Common Options](../common-options/source-common-options.md) for details. [...]
| quote_char | string | no | " | A single character that encloses CSV fields, allowing fields with commas, line breaks, or quotes to be read correctly. [...]
| escape_char | string | no | - | A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. [...]
+| metalake_type | string | no | gravitino | The type of metalake service, currently supports `gravitino`. |
+| recursive_file_scan | boolean | no | true | Whether to scan subdirectories recursively. If `false`, subdirectories will be ignored. [...]
| sort_files_by_modification_time | boolean | no | false | Sort files by modification time in descending order. Enable this when reading evolving schemas to ensure schema inference uses the latest file. |
### file_format_type [string]
@@ -419,6 +421,11 @@ When specified, the connector will fetch table schema from the external metadata
For more information, please refer to [Metadata SPI](../../introduction/concepts/metadata-spi.md).
+### recursive_file_scan [boolean]
+
+Whether to scan subdirectories recursively.
+If `false`, subdirectories will be ignored.
+
## Example
1. In this example, We read data from s3 path `s3a://seatunnel-test/seatunnel/text` and the file type is orc in this path.
diff --git a/docs/en/connectors/source/SftpFile.md b/docs/en/connectors/source/SftpFile.md
index 3e95521f07..92bee70989 100644
--- a/docs/en/connectors/source/SftpFile.md
+++ b/docs/en/connectors/source/SftpFile.md
@@ -121,6 +121,8 @@ The File does not have a specific type list, and we can indicate which SeaTunnel
| file_filter_modified_end | string | no | - | File modification time filter. The connector will filter some files base on the last modification end time (not include end time). The default data format is `yyyy-MM-dd HH:mm:ss`. |
| quote_char | string | no | " | A single character that encloses CSV fields, allowing fields with commas, line breaks, or quotes to be read correctly. |
| escape_char | string | no | - | A single character that allows the quote or other special characters to appear inside a CSV field without ending the field. |
+| metalake_type | string | no | gravitino | The type of metalake service, currently supports `gravitino`. |
+| recursive_file_scan | boolean | no | true | Whether to scan subdirectories recursively. If `false`, subdirectories will be ignored. |
| sort_files_by_modification_time | boolean | no | false | Sort files by modification time in descending order. Enable this when reading evolving schemas to ensure schema inference uses the latest file. |
### file_filter_pattern [string]
@@ -379,6 +381,11 @@ A single character that encloses CSV fields, allowing fields with commas, line b
A single character that allows the quote or other special characters to appear inside a CSV field without ending the field.
+### recursive_file_scan [boolean]
+
+Whether to scan subdirectories recursively.
+If `false`, subdirectories will be ignored.
+
### sort_files_by_modification_time [boolean]
Whether to sort files by modification time in descending order. Default is `false`.
diff --git a/docs/zh/connectors/source/CosFile.md b/docs/zh/connectors/source/CosFile.md
index f9622a1ab3..6841dcbd4e 100644
--- a/docs/zh/connectors/source/CosFile.md
+++ b/docs/zh/connectors/source/CosFile.md
@@ -84,6 +84,7 @@ import ChangeLog from '../changelog/connector-file-cos.md';
| file_filter_modified_end | string | 否 | - |
| quote_char | string | 否 | " |
| escape_char | string | 否 | - |
+| recursive_file_scan | boolean | 否 | true |
| sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后,在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 |
### path [string]
@@ -431,10 +432,14 @@ abc.*
用于在 CSV 字段内转义引号或其他特殊字符,使其不会结束字段。
+### recursive_file_scan [boolean]
+
+是否递归扫描子目录。
+如果设置为 `false`,将忽略子目录,仅扫描指定路径下的文件。
+
### sort_files_by_modification_time [boolean]
是否按修改时间降序排序文件。默认值为 `false`。
-
启用后,文件将按修改时间排序(最新的在前)。适用于以下场景:
- 读取具有不断演化的 schema 的文件,且希望 schema 推断使用最新的文件
- 需要按时间顺序处理文件
diff --git a/docs/zh/connectors/source/FtpFile.md b/docs/zh/connectors/source/FtpFile.md
index 00cffcdfd4..088f1c5783 100644
--- a/docs/zh/connectors/source/FtpFile.md
+++ b/docs/zh/connectors/source/FtpFile.md
@@ -85,6 +85,8 @@ import ChangeLog from '../changelog/connector-file-ftp.md';
| file_filter_modified_end | string | 否 | - |
| quote_char | string | 否 | " |
| escape_char | string | 否 | - |
+| metalake_type | string | 否 | gravitino |
+| recursive_file_scan | boolean | 否 | true |
| sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后,在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 |
### host [string]
@@ -508,10 +510,14 @@ compare_mode = "len_mtime"
用于在 CSV 字段内转义引号或其他特殊字符,使其不会结束字段。
+### recursive_file_scan [boolean]
+
+是否递归扫描子目录。
+如果设置为 `false`,将忽略子目录,仅扫描指定路径下的文件。
+
### sort_files_by_modification_time [boolean]
是否按修改时间降序排序文件。默认值为 `false`。
-
启用后,文件将按修改时间排序(最新的在前)。适用于以下场景:
- 读取具有不断演化的 schema 的文件,且希望 schema 推断使用最新的文件
- 需要按时间顺序处理文件
diff --git a/docs/zh/connectors/source/HdfsFile.md b/docs/zh/connectors/source/HdfsFile.md
index e002cafec6..48814806db 100644
--- a/docs/zh/connectors/source/HdfsFile.md
+++ b/docs/zh/connectors/source/HdfsFile.md
@@ -95,6 +95,8 @@ import ChangeLog from '../changelog/connector-file-hadoop.md';
| file_split_size | long | 否 | 134217728 | `enable_file_split=true` 时生效,单位字节。`text`/`csv`/`json` 按 `file_split_size` 拆分并对齐到下一个 `row_delimiter`;`parquet` 以 RowGroup 为拆分单位,不会切开 RowGroup。 |
| quote_char | string | 否 | " | 用于包裹 CSV 字段的单字符,可保证包含逗号、换行符或引号的字段被正确解析。 |
| escape_char | string | 否 | - | 用于在 CSV 字段内转义引号或其他特殊字符,使其不会结束字段。 |
+| metalake_type | string | 否 | gravitino | Metalake 服务类型,目前支持 `gravitino`。 |
+| recursive_file_scan | boolean | 否 | true | 是否递归扫描子目录。 如果设置为 `false`,将忽略子目录,仅扫描指定路径下的文件。 |
| sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后,在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 |
### file_format_type [string]
@@ -346,6 +348,11 @@ abc.*
更多信息请参考 [元数据 SPI](../../introduction/concepts/metadata-spi.md)。
+### recursive_file_scan [boolean]
+
+是否递归扫描子目录。
+如果设置为 `false`,将忽略子目录,仅扫描指定路径下的文件。
+
### 提示
> 如果您使用 spark/flink,为了使用此连接器,您必须确保您的 spark/flink 集群已经集成了 hadoop。测试过的 hadoop 版本是 2.x。如果您使用 SeaTunnel Engine,则在下载和安装 SeaTunnel Engine 时会自动集成 hadoop jar。您可以检查 `${SEATUNNEL_HOME}/lib` 下的 jar 包来确认这一点。
diff --git a/docs/zh/connectors/source/LocalFile.md b/docs/zh/connectors/source/LocalFile.md
index 323171bbde..a6c4b1ce9a 100644
--- a/docs/zh/connectors/source/LocalFile.md
+++ b/docs/zh/connectors/source/LocalFile.md
@@ -92,6 +92,8 @@ import ChangeLog from '../changelog/connector-file-local.md';
| file_split_size | long | 否 | 134217728 |
| quote_char | string | 否 | - |
| escape_char | string | 否 | - |
+| metalake_type | string | 否 | gravitino | Metalake 服务类型,目前支持 `gravitino`。 |
+| recursive_file_scan | boolean | 否 | true |
| sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后,在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 |
### path [string]
@@ -536,10 +538,14 @@ compare_mode = "len_mtime"
用于在 CSV 字段内转义引号或其他特殊字符,使其不会结束字段。
+### recursive_file_scan [boolean]
+
+是否递归扫描子目录。
+如果设置为 `false`,将忽略子目录,仅扫描指定路径下的文件。
+
### sort_files_by_modification_time [boolean]
是否按修改时间降序排序文件。默认值为 `false`。
-
启用后,文件将按修改时间排序(最新的在前)。适用于以下场景:
- 读取具有不断演化的 schema 的文件,且希望 schema 推断使用最新的文件
- 需要按时间顺序处理文件
diff --git a/docs/zh/connectors/source/ObsFile.md b/docs/zh/connectors/source/ObsFile.md
index 4a44ea4868..e06e48791e 100644
--- a/docs/zh/connectors/source/ObsFile.md
+++ b/docs/zh/connectors/source/ObsFile.md
@@ -80,6 +80,7 @@ import ChangeLog from '../changelog/connector-file-obs.md';
| time_format | string | 否 | HH:mm:ss | 时间类型格式 |
| quote_char | string | 否 | " | 用于包裹 CSV 字段的单字符,可保证包含逗号、换行符或引号的字段被正确解析。 |
| escape_char | string | 否 | - | 用于在 CSV 字段内转义引号或其他特殊字符,使其不会结束字段。 |
+| recursive_file_scan | boolean | 否 | true | 是否递归扫描子目录。 如果设置为 `false`,将忽略子目录,仅扫描指定路径下的文件。 |
| sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后,在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 |
### file_format_type [string]
@@ -112,6 +113,7 @@ markdown 解析器提取各种元素,包括标题、段落、列表、代码
注意:Markdown 格式仅支持读取,不支持写入。
### sort_files_by_modification_time [boolean]
+
是否按修改时间降序排序文件。默认值为 `false`。
启用后,文件将按修改时间排序(最新的在前)。适用于以下场景:
@@ -121,4 +123,3 @@ markdown 解析器提取各种元素,包括标题、段落、列表、代码
## 变更日志
<ChangeLog />
-
diff --git a/docs/zh/connectors/source/OssFile.md b/docs/zh/connectors/source/OssFile.md
index 67cc8a2904..b2018d9cb1 100644
--- a/docs/zh/connectors/source/OssFile.md
+++ b/docs/zh/connectors/source/OssFile.md
@@ -216,6 +216,8 @@ schema {
| file_filter_modified_end | string | 否 | - | 按照最后修改时间过滤文件。 要过滤的结束时间(不包括改时间),时间格式是:`yyyy-MM-dd HH:mm:ss` |
| quote_char | string | 否 | " | 用于包裹 CSV 字段的单字符,可保证包含逗号、换行符或引号的字段被正确解析。 |
| escape_char | string | 否 | - | 用于在 CSV 字段内转义引号或其他特殊字符,使其不会结束字段。 |
+| metalake_type | string | 否 | gravitino | Metalake 服务类型,目前支持 `gravitino`。 |
+| recursive_file_scan | boolean | 否 | true | 是否递归扫描子目录。 如果设置为 `false`,将忽略子目录,仅扫描指定路径下的文件。 |
| sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后,在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 |
### compress_codec [string]
@@ -282,10 +284,14 @@ markdown 解析器提取各种元素,包括标题、段落、列表、代码
用于在 CSV 字段内转义引号或其他特殊字符,使其不会结束字段。
+### recursive_file_scan [boolean]
+
+是否递归扫描子目录。
+如果设置为 `false`,将忽略子目录,仅扫描指定路径下的文件。
+
### sort_files_by_modification_time [boolean]
是否按修改时间降序排序文件。默认值为 `false`。
-
启用后,文件将按修改时间排序(最新的在前)。适用于以下场景:
- 读取具有不断演化的 schema 的文件,且希望 schema 推断使用最新的文件
- 需要按时间顺序处理文件
diff --git a/docs/zh/connectors/source/OssJindoFile.md b/docs/zh/connectors/source/OssJindoFile.md
index 7e68d61b21..a7559992fc 100644
--- a/docs/zh/connectors/source/OssJindoFile.md
+++ b/docs/zh/connectors/source/OssJindoFile.md
@@ -80,12 +80,12 @@ import ChangeLog from '../changelog/connector-file-oss-jindo.md';
| file_filter_pattern | string | 否 | - | 文件过滤模式 |
| quote_char | string | 否 | " | 用于包裹 CSV 字段的单字符,可保证包含逗号、换行符或引号的字段被正确解析。 |
| escape_char | string | 否 | - | 用于在 CSV 字段内转义引号或其他特殊字符,使其不会结束字段。 |
+| recursive_file_scan | boolean | 否 | true | 是否递归扫描子目录。 如果设置为 `false`,将忽略子目录,仅扫描指定路径下的文件。 |
| sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后,在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 |
### sort_files_by_modification_time [boolean]
是否按修改时间降序排序文件。默认值为 `false`。
-
启用后,文件将按修改时间排序(最新的在前)。适用于以下场景:
- 读取具有不断演化的 schema 的文件,且希望 schema 推断使用最新的文件
- 需要按时间顺序处理文件
@@ -122,4 +122,3 @@ markdown 解析器提取各种元素,包括标题、段落、列表、代码
## 变更日志
<ChangeLog />
-
diff --git a/docs/zh/connectors/source/S3File.md b/docs/zh/connectors/source/S3File.md
index 6af96d6202..e91b7a0758 100644
--- a/docs/zh/connectors/source/S3File.md
+++ b/docs/zh/connectors/source/S3File.md
@@ -225,6 +225,8 @@ schema {
| common-options | | 否 | - | 数据源插件通用参数,请参考[数据源通用选项](../common-options/source-common-options.md)了解详情。 |
| quote_char | string | 否 | " | 用于包裹 CSV 字段的单字符,可保证包含逗号、换行符或引号的字段被正确解析。 |
| escape_char | string | 否 | - | 用于在 CSV 字段内转义引号或其他特殊字符,使其不会结束字段。 |
+| metalake_type | string | 否 | gravitino | Metalake 服务类型,目前支持 `gravitino`。 |
+| recursive_file_scan | boolean | 否 | true | 是否递归扫描子目录。 如果设置为 `false`,将忽略子目录,仅扫描指定路径下的文件。 |
| sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后,在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 |
### delimiter/field_delimiter [string]
@@ -247,10 +249,14 @@ schema {
用于在 CSV 字段内转义引号或其他特殊字符,使其不会结束字段。
+### recursive_file_scan [boolean]
+
+是否递归扫描子目录。
+如果设置为 `false`,将忽略子目录,仅扫描指定路径下的文件。
+
### sort_files_by_modification_time [boolean]
是否按修改时间降序排序文件。默认值为 `false`。
-
启用后,文件将按修改时间排序(最新的在前)。适用于以下场景:
- 读取具有不断演化的 schema 的文件,且希望 schema 推断使用最新的文件
- 需要按时间顺序处理文件
diff --git a/docs/zh/connectors/source/SftpFile.md b/docs/zh/connectors/source/SftpFile.md
index ce71dd1cb5..db3dce2ca7 100644
--- a/docs/zh/connectors/source/SftpFile.md
+++ b/docs/zh/connectors/source/SftpFile.md
@@ -121,6 +121,8 @@ import ChangeLog from '../changelog/connector-file-sftp.md';
| file_filter_modified_end | string | 否 | - | 按照最后修改时间过滤文件。 要过滤的结束时间(不包括改时间),时间格式是:`yyyy-MM-dd HH:mm:ss` |
| quote_char | string | 否 | " | 用于包裹 CSV 字段的单字符,可保证包含逗号、换行符或引号的字段被正确解析。 |
| escape_char | string | 否 | - | 用于在 CSV 字段内转义引号或其他特殊字符,使其不会结束字段。 |
+| metalake_type | string | 否 | gravitino | Metalake 服务类型,目前支持 `gravitino`。 |
+| recursive_file_scan | boolean | 否 | true | 是否递归扫描子目录。 如果设置为 `false`,将忽略子目录,仅扫描指定路径下的文件。 |
| sort_files_by_modification_time | boolean | 否 | false | 是否按修改时间降序排序文件。启用此选项后,在读取不断演化的 schema 时可确保 schema 推断使用最新的文件。 |
### file_filter_pattern [string]
@@ -386,10 +388,14 @@ compare_mode = "len_mtime"
更多信息请参考 [元数据 SPI](../../introduction/concepts/metadata-spi.md)。
+### recursive_file_scan [boolean]
+
+是否递归扫描子目录。
+如果设置为 `false`,将忽略子目录,仅扫描指定路径下的文件。
+
### sort_files_by_modification_time [boolean]
是否按修改时间降序排序文件。默认值为 `false`。
-
启用后,文件将按修改时间排序(最新的在前)。适用于以下场景:
- 读取具有不断演化的 schema 的文件,且希望 schema 推断使用最新的文件
- 需要按时间顺序处理文件
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSourceOptions.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSourceOptions.java
index 99ec07fb51..8828eef0c7 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSourceOptions.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSourceOptions.java
@@ -225,6 +225,14 @@ public class FileBaseSourceOptions extends FileBaseOptions {
.withDescription(
"A single character that allows the quote or other special characters to appear inside a CSV field without ending the field.");
+ public static final Option<Boolean> RECURSIVE_FILE_SCAN =
+ Options.key("recursive_file_scan")
+ .booleanType()
+ .defaultValue(true)
+ .withDescription(
+ "Whether to recursively scan subdirectories. "
+ + "If false, subdirectories will be ignored.");
+
public static final Option<Boolean> SORT_FILES_BY_MOD_TIME =
Options.key("sort_files_by_modification_time")
.booleanType()
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java
index 918e4641b5..01cfb40e0a 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java
@@ -131,6 +131,7 @@ public abstract class AbstractReadStrategy implements ReadStrategy {
protected transient HadoopFileSystemProxy targetHadoopFileSystemProxy;
protected transient boolean shareTargetFileSystemProxy;
protected transient boolean checksumUnavailableWarned;
+ protected boolean recursiveFileScan = true;
protected boolean sortFilesByModTime =
FileBaseSourceOptions.SORT_FILES_BY_MOD_TIME.defaultValue();
@@ -190,7 +191,7 @@ public abstract class AbstractReadStrategy implements ReadStrategy {
throws IOException {
FileStatus[] stats = hadoopFileSystemProxy.listStatus(path);
for (FileStatus fileStatus : stats) {
- if (fileStatus.isDirectory()) {
+ if (fileStatus.isDirectory() && recursiveFileScan) {
// skip hidden tmp directory, such as .hive-staging_hive
if (!fileStatus.getPath().getName().startsWith(".")) {
collectFileInfoByPath(
@@ -359,6 +360,12 @@ public abstract class AbstractReadStrategy implements ReadStrategy {
pluginConfig.getString(FileBaseSourceOptions.SYNC_MODE.key()),
FileBaseSourceOptions.SYNC_MODE.key());
}
+
+ if (pluginConfig.hasPath(FileBaseSourceOptions.RECURSIVE_FILE_SCAN.key())) {
+ recursiveFileScan =
+ pluginConfig.getBoolean(FileBaseSourceOptions.RECURSIVE_FILE_SCAN.key());
+ }
+
enableUpdateSync = syncMode == FileSyncMode.UPDATE;
if (enableUpdateSync) {
validateUpdateSyncConfig(pluginConfig);
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/ContinuousMultipleTableFileSourceSplitEnumerator.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/ContinuousMultipleTableFileSourceSplitEnumerator.java
index 93605c4471..34041efcbf 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/ContinuousMultipleTableFileSourceSplitEnumerator.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/ContinuousMultipleTableFileSourceSplitEnumerator.java
@@ -285,7 +285,7 @@ public class ContinuousMultipleTableFileSourceSplitEnumerator
int queued = 0;
Set<String> activeKnownSplitIds = new HashSet<>();
for (TableScanContext ctx : tableScanContexts) {
- List<FileStatus> files = ctx.listFilesRecursively(ctx.rootPath);
+ List<FileStatus> files = ctx.listFiles(ctx.rootPath);
scanned += files.size();
for (FileStatus fileStatus : files) {
if (!ctx.shouldProcess(fileStatus, jobStartTimeMillis, startMode)) {
@@ -526,6 +526,7 @@ public class ContinuousMultipleTableFileSourceSplitEnumerator
private final FileUpdateStrategy updateStrategy;
private final FileCompareMode compareMode;
private boolean checksumUnavailableWarned;
+ private final boolean recursiveFileScan;
private final Pattern pattern;
private final String fileBasePath;
@@ -568,6 +569,7 @@ public class ContinuousMultipleTableFileSourceSplitEnumerator
this.updateStrategy = config.get(FileBaseSourceOptions.UPDATE_STRATEGY);
this.compareMode = config.get(FileBaseSourceOptions.COMPARE_MODE);
+ this.recursiveFileScan = config.get(FileBaseSourceOptions.RECURSIVE_FILE_SCAN);
String targetPath = config.get(FileBaseSourceOptions.TARGET_PATH);
Map<String, String> targetHadoopConf =
@@ -588,14 +590,14 @@ public class ContinuousMultipleTableFileSourceSplitEnumerator
return fileSplitStrategy.split(tableId, fileStatus.getPath().toString());
}
- private List<FileStatus> listFilesRecursively(String path) throws IOException {
+ private List<FileStatus> listFiles(String path) throws IOException {
List<FileStatus> files = new ArrayList<>();
FileStatus[] statuses = sourceFs.listStatus(path);
for (FileStatus status : statuses) {
if (status.isDirectory()) {
String name = status.getPath().getName();
- if (!name.startsWith(".")) {
- files.addAll(listFilesRecursively(status.getPath().toString()));
+ if (recursiveFileScan && !name.startsWith(".")) {
+ files.addAll(listFiles(status.getPath().toString()));
}
continue;
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategyTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategyTest.java
index 69481f30d5..a8b74b2818 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategyTest.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategyTest.java
@@ -47,6 +47,8 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.DisabledOnOs;
import org.junit.jupiter.api.condition.OS;
+import lombok.extern.slf4j.Slf4j;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -61,6 +63,7 @@ import java.util.Map;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
+@Slf4j
public class AbstractReadStrategyTest {
@Test
@@ -505,4 +508,169 @@ public class AbstractReadStrategyTest {
SeaTunnelRowType actualRowType = readStrategy.getActualSeaTunnelRowTypeInfo();
Assertions.assertArrayEquals(new String[] {"id", "dt"}, actualRowType.getFieldNames());
}
+
+ @DisabledOnOs(OS.WINDOWS)
+ @Test
+ public void testNonRecursiveFileScan() throws Exception {
+ String baseDir = "/tmp/test_recursive";
+ String file1 = baseDir + "/file1.txt";
+ String file2 = baseDir + "/file2.txt";
+ String subdirFile = baseDir + "/subdir/file3.txt";
+
+ try {
+ createTestFiles(file1, file2, subdirFile);
+
+ Map<String, Object> config = new HashMap<>();
+ config.put(FileBaseSourceOptions.FILE_PATH.key(), baseDir);
+ config.put(FileBaseSourceOptions.FILE_FORMAT_TYPE.key(), "text");
+ config.put(FileBaseSourceOptions.RECURSIVE_FILE_SCAN.key(), false);
+
+ Config pluginConfig = ConfigFactory.parseMap(config);
+
+ try (TextReadStrategy strategy = new TextReadStrategy()) {
+ LocalFileSystemConf.LocalConf localConf =
+ new LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT);
+ strategy.init(localConf);
+ strategy.setPluginConfig(pluginConfig);
+
+ List<String> fileNames = strategy.getFileNamesByPath(baseDir);
+
+ Assertions.assertEquals(2, fileNames.size());
+ Assertions.assertTrue(fileNames.stream().noneMatch(f -> f.contains("subdir")));
+ Assertions.assertTrue(fileNames.stream().anyMatch(f -> f.endsWith("file1.txt")));
+ Assertions.assertTrue(fileNames.stream().anyMatch(f -> f.endsWith("file2.txt")));
+ }
+
+ } finally {
+ deleteTestDirectory(baseDir);
+ }
+ }
+
+ @DisabledOnOs(OS.WINDOWS)
+ @Test
+ public void testRecursiveFileScanDefault() throws Exception {
+ String baseDir = "/tmp/test_default";
+ String file1 = baseDir + "/file1.txt";
+ String subdirFile = baseDir + "/subdir/file2.txt";
+
+ try {
+ createTestFiles(file1, subdirFile);
+
+ Map<String, Object> config = new HashMap<>();
+ config.put(FileBaseSourceOptions.FILE_PATH.key(), baseDir);
+ config.put(FileBaseSourceOptions.FILE_FORMAT_TYPE.key(), "text");
+
+ Config pluginConfig = ConfigFactory.parseMap(config);
+
+ try (TextReadStrategy strategy = new TextReadStrategy()) {
+ LocalFileSystemConf.LocalConf localConf =
+ new LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT);
+ strategy.init(localConf);
+ strategy.setPluginConfig(pluginConfig);
+
+ List<String> fileNames = strategy.getFileNamesByPath(baseDir);
+
+ Assertions.assertEquals(2, fileNames.size());
+ Assertions.assertTrue(fileNames.stream().anyMatch(f -> f.contains("file1.txt")));
+ Assertions.assertTrue(fileNames.stream().anyMatch(f -> f.contains("subdir")));
+ Assertions.assertTrue(
+ fileNames.stream().anyMatch(f -> f.contains("subdir/file2.txt")));
+ }
+ } finally {
+ deleteTestDirectory(baseDir);
+ }
+ }
+
+ @DisabledOnOs(OS.WINDOWS)
+ @Test
+ public void testNonRecursiveEmptyDirectory() throws Exception {
+ String baseDir = "/tmp/test_empty";
+ String subdirDir = baseDir + "/subdir";
+ String subdirFile = baseDir + "/subdir/file.txt";
+ createTestFiles(subdirFile);
+
+ try {
+ Configuration conf = new Configuration();
+ Path path = new Path(subdirDir);
+ org.apache.hadoop.fs.FileSystem fs = path.getFileSystem(conf);
+ fs.mkdirs(path);
+
+ Map<String, Object> config = new HashMap<>();
+ config.put(FileBaseSourceOptions.FILE_PATH.key(), baseDir);
+ config.put(FileBaseSourceOptions.FILE_FORMAT_TYPE.key(), "text");
+ config.put(FileBaseSourceOptions.RECURSIVE_FILE_SCAN.key(), false);
+
+ Config pluginConfig = ConfigFactory.parseMap(config);
+
+ try (TextReadStrategy strategy = new TextReadStrategy()) {
+ LocalFileSystemConf.LocalConf localConf =
+ new LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT);
+ strategy.init(localConf);
+ strategy.setPluginConfig(pluginConfig);
+
+ List<String> fileNames = strategy.getFileNamesByPath(baseDir);
+
+ Assertions.assertEquals(0, fileNames.size());
+ }
+ } finally {
+ deleteTestDirectory(baseDir);
+ }
+ }
+
+ @DisabledOnOs(OS.WINDOWS)
+ @Test
+ public void testNonRecursiveWithOnlySubdirectories() throws Exception {
+ String baseDir = "/tmp/test_only_subdir";
+ String subdirFile1 = baseDir + "/subdir1/file1.txt";
+ String subdirFile2 = baseDir + "/subdir2/file2.txt";
+
+ try {
+ createTestFiles(subdirFile1, subdirFile2);
+
+ Map<String, Object> config = new HashMap<>();
+ config.put(FileBaseSourceOptions.FILE_PATH.key(), baseDir);
+ config.put(FileBaseSourceOptions.FILE_FORMAT_TYPE.key(), "text");
+ config.put(FileBaseSourceOptions.RECURSIVE_FILE_SCAN.key(), false);
+
+ Config pluginConfig = ConfigFactory.parseMap(config);
+
+ try (TextReadStrategy strategy = new TextReadStrategy()) {
+ LocalFileSystemConf.LocalConf localConf =
+ new LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT);
+ strategy.init(localConf);
+ strategy.setPluginConfig(pluginConfig);
+
+ List<String> fileNames = strategy.getFileNamesByPath(baseDir);
+
+ Assertions.assertEquals(0, fileNames.size());
+ }
+ } finally {
+ deleteTestDirectory(baseDir);
+ }
+ }
+
+ private void createTestFiles(String... filePaths) throws IOException {
+ Configuration conf = new Configuration();
+ for (String filePath : filePaths) {
+ Path path = new Path(filePath);
+ org.apache.hadoop.fs.FileSystem fs = path.getFileSystem(conf);
+
+ fs.mkdirs(path.getParent());
+
+ try (org.apache.hadoop.fs.FSDataOutputStream out = fs.create(path)) {
+ out.writeBytes("test content");
+ }
+ }
+ }
+
+ private void deleteTestDirectory(String dirPath) {
+ try {
+ Path path = new Path(dirPath);
+ Configuration conf = new Configuration();
+ org.apache.hadoop.fs.FileSystem fs = path.getFileSystem(conf);
+ fs.delete(path, true);
+ } catch (Exception e) {
+ log.error("Warning: Failed to delete test directory: " + dirPath, e);
+ }
+ }
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/UpdateSyncModeTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/UpdateSyncModeTest.java
index fff09538f5..d058b707d4 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/UpdateSyncModeTest.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/UpdateSyncModeTest.java
@@ -203,6 +203,73 @@ class UpdateSyncModeTest {
}
}
+ @Test
+ void testUpdateModeNonRecursiveScanOnlyComparesTopLevelFiles() throws Exception {
+ Path sourceDir = tempDir.resolve("src");
+ Path targetDir = tempDir.resolve("dst");
+ Path topLevelSourceFile = sourceDir.resolve("root.bin");
+ Path nestedSourceFile = sourceDir.resolve("subdir/nested.bin");
+ Path topLevelTargetFile = targetDir.resolve("root.bin");
+ Path nestedTargetFile = targetDir.resolve("subdir/nested.bin");
+
+ writeFile(topLevelSourceFile, "root".getBytes());
+ writeFile(nestedSourceFile, "nested".getBytes());
+ writeFile(topLevelTargetFile, "root".getBytes());
+ writeFile(nestedTargetFile, "nested".getBytes());
+ setMtime(topLevelSourceFile, 2_000);
+ setMtime(topLevelTargetFile, 1_000);
+ setMtime(nestedSourceFile, 2_000);
+ setMtime(nestedTargetFile, 1_000);
+
+ try (BinaryReadStrategy strategy = new BinaryReadStrategy()) {
+ strategy.setPluginConfig(
+ updateConfig(
+ sourceDir.toUri().toString(),
+ targetDir.toUri().toString(),
+ "distcp",
+ "len_mtime",
+ false));
+ strategy.init(new LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT));
+
+ List<String> files = strategy.getFileNamesByPath(sourceDir.toUri().toString());
+ Assertions.assertEquals(1, files.size());
+ Assertions.assertTrue(files.get(0).endsWith("/root.bin"));
+ }
+ }
+
+ @Test
+ void testUpdateModeNonRecursiveScanSkipsNestedChanges() throws Exception {
+ Path sourceDir = tempDir.resolve("src");
+ Path targetDir = tempDir.resolve("dst");
+ Path topLevelSourceFile = sourceDir.resolve("root.bin");
+ Path nestedSourceFile = sourceDir.resolve("subdir/nested.bin");
+ Path topLevelTargetFile = targetDir.resolve("root.bin");
+ Path nestedTargetFile = targetDir.resolve("subdir/nested.bin");
+
+ writeFile(topLevelSourceFile, "root".getBytes());
+ writeFile(nestedSourceFile, "nested".getBytes());
+ writeFile(topLevelTargetFile, "root".getBytes());
+ writeFile(nestedTargetFile, "nested".getBytes());
+ setMtime(topLevelSourceFile, 1_000);
+ setMtime(topLevelTargetFile, 1_000);
+ setMtime(nestedSourceFile, 2_000);
+ setMtime(nestedTargetFile, 1_000);
+
+ try (BinaryReadStrategy strategy = new BinaryReadStrategy()) {
+ strategy.setPluginConfig(
+ updateConfig(
+ sourceDir.toUri().toString(),
+ targetDir.toUri().toString(),
+ "distcp",
+ "len_mtime",
+ false));
+ strategy.init(new LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT));
+
+ List<String> files = strategy.getFileNamesByPath(sourceDir.toUri().toString());
+ Assertions.assertTrue(files.isEmpty(), "Nested-only changes should be skipped");
+ }
+ }
+
private static void writeFile(Path path, byte[] content) throws IOException {
Files.createDirectories(path.getParent());
Files.write(path, content);
@@ -214,6 +281,15 @@ class UpdateSyncModeTest {
private static Config updateConfig(
String sourcePath, String targetPath, String updateStrategy, String compareMode) {
+ return updateConfig(sourcePath, targetPath, updateStrategy, compareMode, true);
+ }
+
+ private static Config updateConfig(
+ String sourcePath,
+ String targetPath,
+ String updateStrategy,
+ String compareMode,
+ boolean recursiveFileScan) {
Map<String, Object> configMap = new HashMap<>();
configMap.put("path", sourcePath);
configMap.put("file_format_type", "binary");
@@ -221,6 +297,7 @@ class UpdateSyncModeTest {
configMap.put("target_path", targetPath);
configMap.put("update_strategy", updateStrategy);
configMap.put("compare_mode", compareMode);
+ configMap.put("recursive_file_scan", recursiveFileScan);
return ConfigFactory.parseMap(configMap);
}
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/ContinuousMultipleTableFileSourceSplitEnumeratorTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/ContinuousMultipleTableFileSourceSplitEnumeratorTest.java
index dd3bcb423a..c9178aa194 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/ContinuousMultipleTableFileSourceSplitEnumeratorTest.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/split/ContinuousMultipleTableFileSourceSplitEnumeratorTest.java
@@ -43,9 +43,12 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.FileTime;
+import java.time.Duration;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
@@ -382,7 +385,7 @@ class ContinuousMultipleTableFileSourceSplitEnumeratorTest {
config.put(FileBaseSourceOptions.TARGET_PATH.key(), dstDir.toString());
config.put(FileBaseSourceOptions.UPDATE_STRATEGY.key(), "distcp");
config.put(FileBaseSourceOptions.COMPARE_MODE.key(), "len_mtime");
- config.put(FileBaseSourceOptions.SCAN_INTERVAL.key(), "0S");
+ config.put(FileBaseSourceOptions.SCAN_INTERVAL.key(), Duration.ZERO);
ReadonlyConfig readonlyConfig = ReadonlyConfig.fromMap(config);
@@ -421,10 +424,93 @@ class ContinuousMultipleTableFileSourceSplitEnumeratorTest {
"continuous mode should require a positive scan_interval");
}
+ @Test
+ void testContinuousDiscoveryWithNonRecursiveFileScan() throws Exception {
+ Path srcDir = Files.createDirectories(tempDir.resolve("src_recursive_disabled"));
+ Path dstDir = Files.createDirectories(tempDir.resolve("dst_recursive_disabled"));
+ Path rootFile = srcDir.resolve("root.bin");
+ Path nestedFile = Files.createDirectories(srcDir.resolve("nested")).resolve("nested.bin");
+ Files.write(rootFile, "abc".getBytes());
+ Files.write(nestedFile, "def".getBytes());
+
+ EnumeratorWithContext enumeratorWithContext = createEnumerator(srcDir, dstDir, false);
+ try {
+ enumeratorWithContext.enumerator.scanOnceForTest();
+ Assertions.assertEquals(
+ 1, enumeratorWithContext.enumerator.currentUnassignedSplitSize());
+
+ List<String> filePaths = assignAndCaptureFilePaths(enumeratorWithContext);
+ Assertions.assertEquals(1, filePaths.size());
+ Assertions.assertTrue(filePaths.get(0).endsWith(rootFile.toString()));
+ } finally {
+ enumeratorWithContext.enumerator.close();
+ }
+ }
+
+ @Test
+ void testContinuousDiscoveryWithDefaultRecursiveFileScan() throws Exception {
+ Path srcDir = Files.createDirectories(tempDir.resolve("src_recursive_default"));
+ Path dstDir = Files.createDirectories(tempDir.resolve("dst_recursive_default"));
+ Path rootFile = srcDir.resolve("root.bin");
+ Path nestedFile = Files.createDirectories(srcDir.resolve("nested")).resolve("nested.bin");
+ Files.write(rootFile, "abc".getBytes());
+ Files.write(nestedFile, "def".getBytes());
+
+ EnumeratorWithContext enumeratorWithContext = createEnumerator(srcDir, dstDir);
+ try {
+ enumeratorWithContext.enumerator.scanOnceForTest();
+ Assertions.assertEquals(
+ 2, enumeratorWithContext.enumerator.currentUnassignedSplitSize());
+
+ List<String> filePaths = assignAndCaptureFilePaths(enumeratorWithContext);
+ Assertions.assertTrue(
+ filePaths.stream().anyMatch(path -> path.endsWith(rootFile.toString())));
+ Assertions.assertTrue(
+ filePaths.stream().anyMatch(path -> path.endsWith(nestedFile.toString())));
+ } finally {
+ enumeratorWithContext.enumerator.close();
+ }
+ }
+
+ @Test
+ void testContinuousDiscoveryWithRecursiveFileScan() throws Exception {
+ Path srcDir = Files.createDirectories(tempDir.resolve("src_recursive_enabled"));
+ Path dstDir = Files.createDirectories(tempDir.resolve("dst_recursive_enabled"));
+ Path rootFile = srcDir.resolve("root.bin");
+ Path nestedFile = Files.createDirectories(srcDir.resolve("nested")).resolve("nested.bin");
+ Files.write(rootFile, "abc".getBytes());
+ Files.write(nestedFile, "def".getBytes());
+
+ EnumeratorWithContext enumeratorWithContext = createEnumerator(srcDir, dstDir, true);
+ try {
+ enumeratorWithContext.enumerator.scanOnceForTest();
+ Assertions.assertEquals(
+ 2, enumeratorWithContext.enumerator.currentUnassignedSplitSize());
+
+ List<String> filePaths = assignAndCaptureFilePaths(enumeratorWithContext);
+ Assertions.assertTrue(
+ filePaths.stream().anyMatch(path -> path.endsWith(rootFile.toString())));
+ Assertions.assertTrue(
+ filePaths.stream().anyMatch(path -> path.endsWith(nestedFile.toString())));
+ } finally {
+ enumeratorWithContext.enumerator.close();
+ }
+ }
+
private EnumeratorWithContext createEnumerator(Path srcDir, Path dstDir) throws IOException {
return createEnumerator(srcDir, dstDir, "earliest");
}
+ private EnumeratorWithContext createEnumerator(
+ Path srcDir, Path dstDir, boolean recursiveFileScan) throws IOException {
+ return createEnumerator(
+ srcDir,
+ dstDir,
+ "earliest",
+ new FileSourceState(Collections.emptySet()),
+ recursiveFileScan);
+ }
+
private EnumeratorWithContext createEnumerator(Path srcDir, Path dstDir, String startMode)
throws IOException {
return createEnumerator(
@@ -434,6 +520,16 @@ class ContinuousMultipleTableFileSourceSplitEnumeratorTest {
private EnumeratorWithContext createEnumerator(
Path srcDir, Path dstDir, String startMode, FileSourceState checkpointState)
throws IOException {
+ return createEnumerator(srcDir, dstDir, startMode, checkpointState, null);
+ }
+
+ private EnumeratorWithContext createEnumerator(
+ Path srcDir,
+ Path dstDir,
+ String startMode,
+ FileSourceState checkpointState,
+ Boolean recursiveFileScan)
+ throws IOException {
Map<String, Object> config = new HashMap<>();
config.put(FileBaseSourceOptions.FILE_PATH.key(), srcDir.toString());
config.put(FileBaseSourceOptions.FILE_FORMAT_TYPE.key(), "binary");
@@ -443,6 +539,9 @@ class ContinuousMultipleTableFileSourceSplitEnumeratorTest {
config.put(FileBaseSourceOptions.TARGET_PATH.key(), dstDir.toString());
config.put(FileBaseSourceOptions.UPDATE_STRATEGY.key(), "distcp");
config.put(FileBaseSourceOptions.COMPARE_MODE.key(), "len_mtime");
+ if (recursiveFileScan != null) {
+ config.put(FileBaseSourceOptions.RECURSIVE_FILE_SCAN.key(), recursiveFileScan);
+ }
ReadonlyConfig readonlyConfig = ReadonlyConfig.fromMap(config);
@@ -478,6 +577,19 @@ class ContinuousMultipleTableFileSourceSplitEnumeratorTest {
return new EnumeratorWithContext(enumerator, context);
}
+ private static List<String> assignAndCaptureFilePaths(
+ EnumeratorWithContext enumeratorWithContext) {
+ enumeratorWithContext.enumerator.handleSplitRequest(0);
+ ArgumentCaptor<java.util.List<FileSourceSplit>> splitsCaptor =
+ ArgumentCaptor.forClass((Class) java.util.List.class);
+ Mockito.verify(enumeratorWithContext.context)
+ .assignSplit(Mockito.eq(0), splitsCaptor.capture());
+ return splitsCaptor.getValue().stream()
+ .map(FileSourceSplit::getFilePath)
+ .sorted()
+ .collect(Collectors.toList());
+ }
+
private static final class EnumeratorWithContext {
private final ContinuousMultipleTableFileSourceSplitEnumerator enumerator;
private final SourceSplitEnumerator.Context<FileSourceSplit> context;
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-cos/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/cos/source/CosFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-cos/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/cos/source/CosFileSourceFactory.java
index 68704b920e..50dfd7e91e 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-cos/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/cos/source/CosFileSourceFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-cos/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/cos/source/CosFileSourceFactory.java
@@ -101,6 +101,7 @@ public class CosFileSourceFactory implements TableSourceFactory {
FileBaseSourceOptions.MARKDOWN_RAG_METADATA_ENABLED)
.optional(FileBaseSourceOptions.QUOTE_CHAR)
.optional(FileBaseSourceOptions.ESCAPE_CHAR)
+ .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN)
.build();
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-ftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/ftp/source/FtpFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-ftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/ftp/source/FtpFileSourceFactory.java
index 79499cbf47..4c48a411fb 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-ftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/ftp/source/FtpFileSourceFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-ftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/ftp/source/FtpFileSourceFactory.java
@@ -122,6 +122,7 @@ public class FtpFileSourceFactory implements TableSourceFactory {
FileBaseSourceOptions.SYNC_MODE,
FileSyncMode.UPDATE,
FileBaseSourceOptions.TARGET_PATH)
+ .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN)
.build();
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-hadoop/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/hdfs/source/HdfsFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-hadoop/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/hdfs/source/HdfsFileSourceFactory.java
index de9a088b7e..215a8b8b74 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-hadoop/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/hdfs/source/HdfsFileSourceFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-hadoop/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/hdfs/source/HdfsFileSourceFactory.java
@@ -25,6 +25,7 @@ import org.apache.seatunnel.api.table.connector.TableSource;
import org.apache.seatunnel.api.table.factory.Factory;
import org.apache.seatunnel.api.table.factory.TableSourceFactory;
import org.apache.seatunnel.api.table.factory.TableSourceFactoryContext;
+import org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSourceOptions;
import org.apache.seatunnel.connectors.seatunnel.file.config.FileFormat;
import org.apache.seatunnel.connectors.seatunnel.file.config.FileSyncMode;
import org.apache.seatunnel.connectors.seatunnel.file.config.FileSystemType;
@@ -133,6 +134,7 @@ public class HdfsFileSourceFactory implements TableSourceFactory {
.optional(HdfsFileSourceOptions.QUOTE_CHAR)
.optional(HdfsFileSourceOptions.ESCAPE_CHAR)
.optional(ConnectorCommonOptions.METALAKE_TYPE)
+ .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN)
.build();
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-jindo-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/jindo/source/OssFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-jindo-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/jindo/source/OssFileSourceFactory.java
index d07ab8fffe..75939c1ffb 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-jindo-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/jindo/source/OssFileSourceFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-jindo-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/jindo/source/OssFileSourceFactory.java
@@ -96,6 +96,7 @@ public class OssFileSourceFactory implements TableSourceFactory {
FileBaseSourceOptions.MARKDOWN_RAG_METADATA_ENABLED)
.optional(FileBaseSourceOptions.QUOTE_CHAR)
.optional(FileBaseSourceOptions.ESCAPE_CHAR)
+ .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN)
.build();
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-local/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/local/source/LocalFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-local/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/local/source/LocalFileSourceFactory.java
index 5cd72b6513..919eaecd31 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-local/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/local/source/LocalFileSourceFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-local/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/local/source/LocalFileSourceFactory.java
@@ -129,6 +129,7 @@ public class LocalFileSourceFactory implements TableSourceFactory {
FileBaseSourceOptions.SYNC_MODE,
FileSyncMode.UPDATE,
FileBaseSourceOptions.TARGET_PATH)
+ .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN)
.build();
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-obs/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/obs/source/ObsFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-obs/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/obs/source/ObsFileSourceFactory.java
index 1f97737952..44e079c6c1 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-obs/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/obs/source/ObsFileSourceFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-obs/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/obs/source/ObsFileSourceFactory.java
@@ -85,6 +85,7 @@ public class ObsFileSourceFactory implements TableSourceFactory {
FileBaseSourceOptions.MARKDOWN_RAG_METADATA_ENABLED)
.optional(FileBaseSourceOptions.QUOTE_CHAR)
.optional(FileBaseSourceOptions.ESCAPE_CHAR)
+ .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN)
.build();
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/source/OssFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/source/OssFileSourceFactory.java
index b620f06289..8fc59b4db2 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/source/OssFileSourceFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/source/OssFileSourceFactory.java
@@ -106,6 +106,7 @@ public class OssFileSourceFactory implements TableSourceFactory {
.optional(FileBaseSourceOptions.QUOTE_CHAR)
.optional(FileBaseSourceOptions.ESCAPE_CHAR)
.optional(ConnectorCommonOptions.METALAKE_TYPE)
+ .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN)
.build();
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-s3/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/s3/source/S3FileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-s3/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/s3/source/S3FileSourceFactory.java
index f982fcce4a..89194eff0c 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-s3/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/s3/source/S3FileSourceFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-s3/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/s3/source/S3FileSourceFactory.java
@@ -122,6 +122,7 @@ public class S3FileSourceFactory implements TableSourceFactory {
.optional(FileBaseSourceOptions.QUOTE_CHAR)
.optional(FileBaseSourceOptions.ESCAPE_CHAR)
.optional(ConnectorCommonOptions.METALAKE_TYPE)
+ .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN)
.build();
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/source/SftpFileSourceFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/source/SftpFileSourceFactory.java
index 4ce72d9eb4..acf72f17e9 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/source/SftpFileSourceFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/source/SftpFileSourceFactory.java
@@ -110,6 +110,7 @@ public class SftpFileSourceFactory implements TableSourceFactory {
FileBaseSourceOptions.SYNC_MODE,
FileSyncMode.UPDATE,
FileBaseSourceOptions.TARGET_PATH)
+ .optional(FileBaseSourceOptions.RECURSIVE_FILE_SCAN)
.build();
}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/ftp/FtpFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/ftp/FtpFileIT.java
index da8f533995..8bd3d510ca 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/ftp/FtpFileIT.java
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/ftp/FtpFileIT.java
@@ -171,6 +171,26 @@ public class FtpFileIT extends TestSuiteBase implements TestResource {
ContainerUtil.copyFileIntoContainers(
"/excel/e2e.xlsx", ftpHomeDir + "/e2e.xlsx", ftpContainer);
+ ContainerUtil.copyFileIntoContainers(
+ "/text/e2e.txt",
+ ftpHomeDir + "/tmp/seatunnel/read/recursive/e2e.txt",
+ ftpContainer);
+
+ ContainerUtil.copyFileIntoContainers(
+ "/text/e2e.txt",
+ ftpHomeDir + "/tmp/seatunnel/read/recursive/subdir/e2e.txt",
+ ftpContainer);
+
+ ContainerUtil.copyFileIntoContainers(
+ "/text/e2e.txt",
+ ftpHomeDir + "/tmp/seatunnel/read/recursive/subdir/deeper/e2e.txt",
+ ftpContainer);
+
+ ContainerUtil.copyFileIntoContainers(
+ "/text/e2e.txt",
+ ftpHomeDir + "/tmp/seatunnel/read/recursive/subdir/deeper/final/e2e.txt",
+ ftpContainer);
+
ftpContainer.execInContainer("sh", "-c", "chmod -R 777 " + ftpHomeDir + "/");
ftpContainer.execInContainer("sh", "-c", "chown -R ftp:ftp " + ftpHomeDir + "/");
}
@@ -302,6 +322,95 @@ public class FtpFileIT extends TestSuiteBase implements TestResource {
deleteFileFromContainer(ftpHomeDir + "/tmp/seatunnel/continuous");
}
+ @TestTemplate
+ @DisabledOnContainer(
+ value = {},
+ type = {EngineType.FLINK, EngineType.SPARK},
+ disabledReason = "Continuous discovery is a long-running job; only run in zeta engine.")
+ public void testFtpBinaryUpdateModeContinuousDiscoveryWithNonRecursiveScan(
+ TestContainer container) throws IOException, InterruptedException {
+ resetContinuousTestPath();
+
+ String jobId = String.valueOf(JobIdGenerator.newJobId());
+ CompletableFuture<Container.ExecResult> jobFuture =
+ CompletableFuture.supplyAsync(
+ () -> {
+ try {
+ return container.executeJob(
+ "/text/ftp_binary_update_distcp_continuous_non_recursive.conf",
+ jobId);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ putFtpFile("/tmp/seatunnel/continuous/src/root.bin", "root");
+ putFtpFile("/tmp/seatunnel/continuous/src/subdir/nested.bin", "nested");
+
+ Awaitility.await()
+ .atMost(60, TimeUnit.SECONDS)
+ .untilAsserted(
+ () ->
+ Assertions.assertEquals(
+ "root",
+ readFtpFile("/tmp/seatunnel/continuous/dst/root.bin")));
+
+ Thread.sleep(3000);
+ Assertions.assertFalse(isFtpFileExists("/tmp/seatunnel/continuous/dst/subdir/nested.bin"));
+
+ Container.ExecResult cancelResult = container.cancelJob(jobId);
+ Assertions.assertEquals(0, cancelResult.getExitCode(), cancelResult.getStderr());
+
+ Container.ExecResult execResult;
+ try {
+ execResult = jobFuture.get(120, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new RuntimeException("Wait continuous job exit failed.", e);
+ }
+ Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr());
+
+ deleteFileFromContainer(ftpHomeDir + "/tmp/seatunnel/continuous");
+ }
+
+ @TestTemplate
+ public void testFtpBinaryUpdateModeDistcpWithNonRecursiveScan(TestContainer container)
+ throws IOException, InterruptedException {
+ resetUpdateTestPath();
+ putFtpFile("/tmp/seatunnel/update/src/root.bin", "root-updated-v2");
+ putFtpFile("/tmp/seatunnel/update/src/subdir/nested.bin", "nest-updated-v2");
+ putFtpFile("/tmp/seatunnel/update/dst/root.bin", "root-stale-v1");
+ putFtpFile("/tmp/seatunnel/update/dst/subdir/nested.bin", "nest-stale-v1");
+
+ Container.ExecResult execResult =
+ container.executeJob("/text/ftp_binary_update_non_recursive_distcp.conf");
+ Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr());
+ Assertions.assertEquals(
+ "root-updated-v2", readFtpFile("/tmp/seatunnel/update/dst/root.bin"));
+ Assertions.assertEquals(
+ "nest-stale-v1", readFtpFile("/tmp/seatunnel/update/dst/subdir/nested.bin"));
+
+ deleteFileFromContainer(ftpHomeDir + "/tmp/seatunnel/update");
+ }
+
+ @TestTemplate
+ public void testFtpBinaryUpdateModeStrictChecksumSkipsNestedChangesWithNonRecursiveScan(
+ TestContainer container) throws IOException, InterruptedException {
+ resetUpdateTestPath();
+ putFtpFile("/tmp/seatunnel/update/src/root.bin", "root-same-v1");
+ putFtpFile("/tmp/seatunnel/update/src/subdir/nested.bin", "nest-new-v1");
+ putFtpFile("/tmp/seatunnel/update/dst/root.bin", "root-same-v1");
+ putFtpFile("/tmp/seatunnel/update/dst/subdir/nested.bin", "nest-old-v1");
+
+ Container.ExecResult execResult =
+ container.executeJob("/text/ftp_binary_update_non_recursive_strict_checksum.conf");
+ Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr());
+ Assertions.assertEquals("root-same-v1", readFtpFile("/tmp/seatunnel/update/dst/root.bin"));
+ Assertions.assertEquals(
+ "nest-old-v1", readFtpFile("/tmp/seatunnel/update/dst/subdir/nested.bin"));
+
+ deleteFileFromContainer(ftpHomeDir + "/tmp/seatunnel/update");
+ }
+
@TestTemplate
public void testFtpToAssertForJsonFilter(TestContainer container)
throws IOException, InterruptedException {
@@ -391,14 +500,9 @@ public class FtpFileIT extends TestSuiteBase implements TestResource {
helper.execute("/excel/fake_source_to_ftp_root_path_excel.conf");
// test ftp source support multipleTable
- String homePath = ftpHomeDir;
- String sink01 = "/tmp/seatunnel/json/sink/multiplesource/fake01";
- String sink02 = "/tmp/seatunnel/json/sink/multiplesource/fake02";
- deleteFileFromContainer(homePath + sink01);
- deleteFileFromContainer(homePath + sink02);
- helper.execute("/json/ftp_file_json_to_assert_with_multipletable.conf");
- Assertions.assertEquals(getFileListFromContainer(homePath + sink01).size(), 1);
- Assertions.assertEquals(getFileListFromContainer(homePath + sink02).size(), 1);
+ // test read recursive file path
+ helper.execute("/text/ftp_file_text_recursive_to_assert.conf");
+ helper.execute("/text/ftp_file_text_non_recursive_to_assert.conf");
}
@TestTemplate
@@ -550,6 +654,13 @@ public class FtpFileIT extends TestSuiteBase implements TestResource {
return catResult.getStdout() == null ? "" : catResult.getStdout().trim();
}
+ private boolean isFtpFileExists(String ftpPath) throws IOException, InterruptedException {
+ String containerPath = ftpHomeDir + ftpPath;
+ Container.ExecResult result =
+ ftpContainer.execInContainer("sh", "-c", "test -f '" + containerPath + "'");
+ return result.getExitCode() == 0;
+ }
+
private long getFtpFileMtimeSeconds(String ftpPath) throws IOException, InterruptedException {
String containerPath = ftpHomeDir + ftpPath;
Container.ExecResult result =
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_distcp_continuous_non_recursive.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_distcp_continuous_non_recursive.conf
new file mode 100644
index 0000000000..3d88995203
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_distcp_continuous_non_recursive.conf
@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "STREAMING"
+ checkpoint.interval = 5000
+}
+
+source {
+ FtpFile {
+ host = "ftp"
+ port = 21
+ user = seatunnel
+ password = pass
+
+ path = "/tmp/seatunnel/continuous/src"
+ file_format_type = "binary"
+
+ discovery_mode = "continuous"
+ scan_interval = "1S"
+ start_mode = "earliest"
+
+ sync_mode = "update"
+ target_path = "/tmp/seatunnel/continuous/dst"
+ update_strategy = "distcp"
+ compare_mode = "len_mtime"
+ recursive_file_scan = false
+ }
+}
+
+sink {
+ FtpFile {
+ host = "ftp"
+ port = 21
+ user = seatunnel
+ password = pass
+
+ path = "/tmp/seatunnel/continuous/dst"
+ tmp_path = "/tmp/seatunnel/continuous/tmp"
+ file_format_type = "binary"
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_non_recursive_distcp.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_non_recursive_distcp.conf
new file mode 100644
index 0000000000..45e1cb414d
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_non_recursive_distcp.conf
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+}
+
+source {
+ FtpFile {
+ host = "ftp"
+ port = 21
+ user = seatunnel
+ password = pass
+
+ path = "/tmp/seatunnel/update/src"
+ file_format_type = "binary"
+
+ sync_mode = "update"
+ target_path = "/tmp/seatunnel/update/dst"
+ update_strategy = "distcp"
+ compare_mode = "len_mtime"
+ recursive_file_scan = false
+ }
+}
+
+sink {
+ FtpFile {
+ host = "ftp"
+ port = 21
+ user = seatunnel
+ password = pass
+
+ path = "/tmp/seatunnel/update/dst"
+ tmp_path = "/tmp/seatunnel/update/tmp"
+ file_format_type = "binary"
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_non_recursive_strict_checksum.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_non_recursive_strict_checksum.conf
new file mode 100644
index 0000000000..e1df788853
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_binary_update_non_recursive_strict_checksum.conf
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+}
+
+source {
+ FtpFile {
+ host = "ftp"
+ port = 21
+ user = seatunnel
+ password = pass
+
+ path = "/tmp/seatunnel/update/src"
+ file_format_type = "binary"
+
+ sync_mode = "update"
+ target_path = "/tmp/seatunnel/update/dst"
+ update_strategy = "strict"
+ compare_mode = "checksum"
+ recursive_file_scan = false
+ }
+}
+
+sink {
+ FtpFile {
+ host = "ftp"
+ port = 21
+ user = seatunnel
+ password = pass
+
+ path = "/tmp/seatunnel/update/dst"
+ tmp_path = "/tmp/seatunnel/update/tmp"
+ file_format_type = "binary"
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_file_text_non_recursive_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_file_text_non_recursive_to_assert.conf
new file mode 100644
index 0000000000..274b0a9154
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_file_text_non_recursive_to_assert.conf
@@ -0,0 +1,121 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+
+ spark.app.name = "SeaTunnel"
+ spark.executor.instances = 1
+ spark.executor.cores = 1
+ spark.executor.memory = "1g"
+ spark.master = local
+}
+
+source {
+ FtpFile {
+ host = "ftp"
+ port = 21
+ user = seatunnel
+ password = pass
+ path = "/tmp/seatunnel/read/recursive"
+ file_format_type = "text"
+ recursive_file_scan = false
+ plugin_output = "ftp"
+ schema = {
+ fields {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ c_row = {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ }
+ }
+ }
+ }
+}
+
+sink {
+ Assert {
+ plugin_input = "ftp"
+ rules {
+ row_rules = [
+ {
+ rule_type = MAX_ROW
+ rule_value = 5
+ },
+ {
+ rule_type = MIN_ROW
+ rule_value = 5
+ }
+ ],
+ field_rules = [
+ {
+ field_name = c_string
+ field_type = string
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = c_boolean
+ field_type = boolean
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = c_double
+ field_type = double
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ }
+ ]
+ }
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_file_text_recursive_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_file_text_recursive_to_assert.conf
new file mode 100644
index 0000000000..3bf4b871fd
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-ftp-e2e/src/test/resources/text/ftp_file_text_recursive_to_assert.conf
@@ -0,0 +1,121 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+
+ spark.app.name = "SeaTunnel"
+ spark.executor.instances = 1
+ spark.executor.cores = 1
+ spark.executor.memory = "1g"
+ spark.master = local
+}
+
+source {
+ FtpFile {
+ host = "ftp"
+ port = 21
+ user = seatunnel
+ password = pass
+ path = "/tmp/seatunnel/read/recursive"
+ file_format_type = "text"
+ recursive_file_scan = true
+ plugin_output = "ftp"
+ schema = {
+ fields {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ c_row = {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ }
+ }
+ }
+ }
+}
+
+sink {
+ Assert {
+ plugin_input = "ftp"
+ rules {
+ row_rules = [
+ {
+ rule_type = MAX_ROW
+ rule_value = 20
+ },
+ {
+ rule_type = MIN_ROW
+ rule_value = 20
+ }
+ ],
+ field_rules = [
+ {
+ field_name = c_string
+ field_type = string
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = c_boolean
+ field_type = boolean
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = c_double
+ field_type = double
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ }
+ ]
+ }
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/hdfs/HdfsFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/hdfs/HdfsFileIT.java
index f8c6d97fe6..0577eaf42d 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/hdfs/HdfsFileIT.java
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/hdfs/HdfsFileIT.java
@@ -23,6 +23,7 @@ import org.apache.seatunnel.e2e.common.container.EngineType;
import org.apache.seatunnel.e2e.common.container.TestContainer;
import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer;
import org.apache.seatunnel.e2e.common.junit.TestContainerExtension;
+import org.apache.seatunnel.e2e.common.util.ContainerUtil;
import org.apache.seatunnel.e2e.common.util.JobIdGenerator;
import org.awaitility.Awaitility;
@@ -170,6 +171,24 @@ public class HdfsFileIT extends TestSuiteBase implements TestResource {
Assertions.assertEquals(0, readResult.getExitCode());
}
+ @TestTemplate
+ public void testHdfsReadRecursiveTextDirectory(TestContainer container)
+ throws IOException, InterruptedException {
+ resetRecursiveTestPath();
+ putHdfsClasspathFile("/text/e2e.txt", "/recursive/e2e.txt");
+ putHdfsClasspathFile("/text/e2e.txt", "/recursive/subdir/e2e.txt");
+ putHdfsClasspathFile("/text/e2e.txt", "/recursive/subdir/deeper/e2e.txt");
+ putHdfsClasspathFile("/text/e2e.txt", "/recursive/subdir/deeper/final/e2e.txt");
+
+ org.testcontainers.containers.Container.ExecResult recursiveResult =
+ container.executeJob("/hdfs_text_recursive_to_assert.conf");
+ Assertions.assertEquals(0, recursiveResult.getExitCode());
+
+ org.testcontainers.containers.Container.ExecResult nonRecursiveResult =
+ container.executeJob("/hdfs_text_non_recursive_to_assert.conf");
+ Assertions.assertEquals(0, nonRecursiveResult.getExitCode());
+ }
+
@TestTemplate
public void testHdfsBinaryUpdateModeDistcp(TestContainer container)
throws IOException, InterruptedException {
@@ -275,6 +294,91 @@ public class HdfsFileIT extends TestSuiteBase implements TestResource {
nameNode.execInContainer("bash", "-c", "hdfs dfs -rm -r -f /continuous || true");
}
+ @TestTemplate
+ @DisabledOnContainer(
+ value = {},
+ type = {EngineType.FLINK, EngineType.SPARK},
+ disabledReason = "Continuous discovery is a long-running job; only run in zeta engine.")
+ public void testHdfsBinaryUpdateModeContinuousDiscoveryWithNonRecursiveScan(
+ TestContainer container) throws IOException, InterruptedException {
+ resetContinuousTestPath();
+
+ String jobId = String.valueOf(JobIdGenerator.newJobId());
+ CompletableFuture<org.testcontainers.containers.Container.ExecResult> jobFuture =
+ CompletableFuture.supplyAsync(
+ () -> {
+ try {
+ return container.executeJob(
+ "/hdfs_binary_update_distcp_continuous_non_recursive.conf",
+ jobId);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ createHdfsDirectories("/continuous/src/subdir");
+ putHdfsFile("/continuous/src/root.bin", "root");
+ putHdfsFile("/continuous/src/subdir/nested.bin", "nested");
+
+ Awaitility.await()
+ .atMost(60, TimeUnit.SECONDS)
+ .untilAsserted(
+ () ->
+ Assertions.assertEquals(
+ "root", readHdfsFile("/continuous/dst/root.bin")));
+
+ Thread.sleep(3000);
+ Assertions.assertFalse(isHdfsFileExists("/continuous/dst/subdir/nested.bin"));
+
+ org.testcontainers.containers.Container.ExecResult cancelResult =
+ container.cancelJob(jobId);
+ Assertions.assertEquals(0, cancelResult.getExitCode(), cancelResult.getStderr());
+
+ org.testcontainers.containers.Container.ExecResult execResult;
+ try {
+ execResult = jobFuture.get(120, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new RuntimeException("Wait continuous job exit failed.", e);
+ }
+ Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr());
+
+ nameNode.execInContainer("bash", "-c", "hdfs dfs -rm -r -f /continuous || true");
+ }
+
+ @TestTemplate
+ public void testHdfsBinaryUpdateModeDistcpWithNonRecursiveScan(TestContainer container)
+ throws IOException, InterruptedException {
+ resetUpdateTestPath();
+ createHdfsDirectories("/update/src/subdir", "/update/dst/subdir");
+ putHdfsFile("/update/src/root.bin", "root-updated-v2");
+ putHdfsFile("/update/src/subdir/nested.bin", "nest-updated-v2");
+ putHdfsFile("/update/dst/root.bin", "root-stale-v1");
+ putHdfsFile("/update/dst/subdir/nested.bin", "nest-stale-v1");
+
+ org.testcontainers.containers.Container.ExecResult execResult =
+ container.executeJob("/hdfs_binary_update_non_recursive_distcp.conf");
+ Assertions.assertEquals(0, execResult.getExitCode());
+ Assertions.assertEquals("root-updated-v2", readHdfsFile("/update/dst/root.bin"));
+ Assertions.assertEquals("nest-stale-v1", readHdfsFile("/update/dst/subdir/nested.bin"));
+ }
+
+ @TestTemplate
+ public void testHdfsBinaryUpdateModeStrictChecksumSkipsNestedChangesWithNonRecursiveScan(
+ TestContainer container) throws IOException, InterruptedException {
+ resetUpdateTestPath();
+ createHdfsDirectories("/update/src/subdir", "/update/dst/subdir");
+ putHdfsFile("/update/src/root.bin", "root-same-v1");
+ putHdfsFile("/update/src/subdir/nested.bin", "nest-new-v1");
+ putHdfsFile("/update/dst/root.bin", "root-same-v1");
+ putHdfsFile("/update/dst/subdir/nested.bin", "nest-old-v1");
+
+ org.testcontainers.containers.Container.ExecResult execResult =
+ container.executeJob("/hdfs_binary_update_non_recursive_strict_checksum.conf");
+ Assertions.assertEquals(0, execResult.getExitCode());
+ Assertions.assertEquals("root-same-v1", readHdfsFile("/update/dst/root.bin"));
+ Assertions.assertEquals("nest-old-v1", readHdfsFile("/update/dst/subdir/nested.bin"));
+ }
+
private void resetUpdateTestPath() throws IOException, InterruptedException {
nameNode.execInContainer("bash", "-c", "hdfs dfs -rm -r -f /update || true");
org.testcontainers.containers.Container.ExecResult mkdirResult =
@@ -304,6 +408,14 @@ public class HdfsFileIT extends TestSuiteBase implements TestResource {
Assertions.assertEquals(0, mkdirResult.getExitCode());
}
+ private void resetRecursiveTestPath() throws IOException, InterruptedException {
+ nameNode.execInContainer("bash", "-c", "hdfs dfs -rm -r -f /recursive || true");
+ org.testcontainers.containers.Container.ExecResult mkdirResult =
+ nameNode.execInContainer(
+ "hdfs", "dfs", "-mkdir", "-p", "/recursive/subdir/deeper/final");
+ Assertions.assertEquals(0, mkdirResult.getExitCode());
+ }
+
private void putHdfsFile(String hdfsPath, String content)
throws IOException, InterruptedException {
String command = "printf '" + content + "' | hdfs dfs -put -f - " + hdfsPath;
@@ -312,6 +424,27 @@ public class HdfsFileIT extends TestSuiteBase implements TestResource {
Assertions.assertEquals(0, putResult.getExitCode());
}
+ private void createHdfsDirectories(String... hdfsPaths)
+ throws IOException, InterruptedException {
+ String command = "hdfs dfs -mkdir -p " + String.join(" ", hdfsPaths);
+ org.testcontainers.containers.Container.ExecResult mkdirResult =
+ nameNode.execInContainer("bash", "-c", command);
+ Assertions.assertEquals(0, mkdirResult.getExitCode());
+ }
+
+ private void putHdfsClasspathFile(String resourcePath, String hdfsPath)
+ throws IOException, InterruptedException {
+ String tempFileName =
+ "/tmp/"
+ + java.util.UUID.randomUUID()
+ + "-"
+ + java.nio.file.Paths.get(resourcePath).getFileName();
+ ContainerUtil.copyFileIntoContainers(resourcePath, tempFileName, nameNode);
+ org.testcontainers.containers.Container.ExecResult putResult =
+ nameNode.execInContainer("hdfs", "dfs", "-put", "-f", tempFileName, hdfsPath);
+ Assertions.assertEquals(0, putResult.getExitCode());
+ }
+
private void putHdfsSequentialLinesFile(String hdfsPath, int lineCount)
throws IOException, InterruptedException {
String command =
@@ -331,6 +464,12 @@ public class HdfsFileIT extends TestSuiteBase implements TestResource {
return catResult.getStdout() == null ? "" : catResult.getStdout().trim();
}
+ private boolean isHdfsFileExists(String hdfsPath) throws IOException, InterruptedException {
+ org.testcontainers.containers.Container.ExecResult result =
+ nameNode.execInContainer("hdfs", "dfs", "-test", "-f", hdfsPath);
+ return result.getExitCode() == 0;
+ }
+
private long getHdfsFileMtimeSeconds(String hdfsPath) throws IOException, InterruptedException {
org.testcontainers.containers.Container.ExecResult statResult =
nameNode.execInContainer("bash", "-c", "hdfs dfs -stat %Y " + hdfsPath);
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_distcp_continuous_non_recursive.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_distcp_continuous_non_recursive.conf
new file mode 100644
index 0000000000..4f91e7407d
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_distcp_continuous_non_recursive.conf
@@ -0,0 +1,58 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "STREAMING"
+ checkpoint.interval = 5000
+}
+
+source {
+ HdfsFile {
+ fs.defaultFS = "hdfs://namenode1:9000"
+ path = "/continuous/src"
+ file_format_type = "binary"
+
+ discovery_mode = "continuous"
+ scan_interval = "1S"
+ start_mode = "earliest"
+
+ sync_mode = "update"
+ target_path = "/continuous/dst"
+ update_strategy = "distcp"
+ compare_mode = "len_mtime"
+ recursive_file_scan = false
+
+ hadoop_conf = {
+ "dfs.replication" = 1
+ }
+ }
+}
+
+sink {
+ HdfsFile {
+ fs.defaultFS = "hdfs://namenode1:9000"
+ path = "/continuous/dst"
+ tmp_path = "/continuous/tmp"
+ file_format_type = "binary"
+ data_save_mode = "APPEND_DATA"
+
+ hadoop_conf = {
+ "dfs.replication" = 1
+ }
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_non_recursive_distcp.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_non_recursive_distcp.conf
new file mode 100644
index 0000000000..8ff41bbedf
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_non_recursive_distcp.conf
@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+}
+
+source {
+ HdfsFile {
+ fs.defaultFS = "hdfs://namenode1:9000"
+ path = "/update/src"
+ file_format_type = "binary"
+
+ sync_mode = "update"
+ target_path = "/update/dst"
+ update_strategy = "distcp"
+ compare_mode = "len_mtime"
+ recursive_file_scan = false
+
+ hadoop_conf = {
+ "dfs.replication" = 1
+ }
+ }
+}
+
+sink {
+ HdfsFile {
+ fs.defaultFS = "hdfs://namenode1:9000"
+ path = "/update/dst"
+ tmp_path = "/update/tmp"
+ file_format_type = "binary"
+ data_save_mode = "APPEND_DATA"
+
+ hadoop_conf = {
+ "dfs.replication" = 1
+ }
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_non_recursive_strict_checksum.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_non_recursive_strict_checksum.conf
new file mode 100644
index 0000000000..cb02dd2909
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_binary_update_non_recursive_strict_checksum.conf
@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+}
+
+source {
+ HdfsFile {
+ fs.defaultFS = "hdfs://namenode1:9000"
+ path = "/update/src"
+ file_format_type = "binary"
+
+ sync_mode = "update"
+ target_path = "/update/dst"
+ update_strategy = "strict"
+ compare_mode = "checksum"
+ recursive_file_scan = false
+
+ hadoop_conf = {
+ "dfs.replication" = 1
+ }
+ }
+}
+
+sink {
+ HdfsFile {
+ fs.defaultFS = "hdfs://namenode1:9000"
+ path = "/update/dst"
+ tmp_path = "/update/tmp"
+ file_format_type = "binary"
+ data_save_mode = "APPEND_DATA"
+
+ hadoop_conf = {
+ "dfs.replication" = 1
+ }
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_text_non_recursive_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_text_non_recursive_to_assert.conf
new file mode 100644
index 0000000000..0ec366364f
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_text_non_recursive_to_assert.conf
@@ -0,0 +1,113 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+}
+
+source {
+ HdfsFile {
+ fs.defaultFS = "hdfs://namenode1:9000"
+ path = "/recursive"
+ file_format_type = "text"
+ recursive_file_scan = false
+ schema = {
+ fields {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ c_row = {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ }
+ }
+ }
+ hadoop_conf = {
+ "dfs.replication" = 1
+ }
+ }
+}
+
+sink {
+ Assert {
+ rules {
+ row_rules = [
+ {
+ rule_type = MAX_ROW
+ rule_value = 5
+ },
+ {
+ rule_type = MIN_ROW
+ rule_value = 5
+ }
+ ],
+ field_rules = [
+ {
+ field_name = c_string
+ field_type = string
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = c_boolean
+ field_type = boolean
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = c_double
+ field_type = double
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ }
+ ]
+ }
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_text_recursive_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_text_recursive_to_assert.conf
new file mode 100644
index 0000000000..f6d713d63e
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/hdfs_text_recursive_to_assert.conf
@@ -0,0 +1,113 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+}
+
+source {
+ HdfsFile {
+ fs.defaultFS = "hdfs://namenode1:9000"
+ path = "/recursive"
+ file_format_type = "text"
+ recursive_file_scan = true
+ schema = {
+ fields {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ c_row = {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ }
+ }
+ }
+ hadoop_conf = {
+ "dfs.replication" = 1
+ }
+ }
+}
+
+sink {
+ Assert {
+ rules {
+ row_rules = [
+ {
+ rule_type = MAX_ROW
+ rule_value = 20
+ },
+ {
+ rule_type = MIN_ROW
+ rule_value = 20
+ }
+ ],
+ field_rules = [
+ {
+ field_name = c_string
+ field_type = string
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = c_boolean
+ field_type = boolean
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = c_double
+ field_type = double
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ }
+ ]
+ }
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/text/e2e.txt b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/text/e2e.txt
new file mode 100644
index 0000000000..9871cd85eb
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-hadoop-e2e/src/test/resources/text/e2e.txt
@@ -0,0 +1,5 @@
+uDDrwsQQYONTNeUBIOnLAgunvDqLBObroRzdEdvDgRmgaeFyFH5456857591576298739157764687713794636442057612252MTDnafalse3313846190943192276641872220071936002.4798444E389.52375328387482E307vcIGF2023-06-0776258155390368615610.7646252373186602912023-05-08 16:08:51ipToEdierOAbwQfQzObWqiRhjkWYaMKdCbjurhstsWrAVlRyyR2905930362869031292782506910815576701385108050hArFutrue12631169122166306155952414159791708165.949173E372.1775762383875058E307kMlgO2023-05-20 [...]
+QIpzzZNFkLwARZDSdwdBzkegCdIRVYJnuXgxNXytAJxxaTzmDF16603816781145850255103997497062535321459349811xaTOktrue5327578191749099325840234439082792961.955231E381.5072154481920294E308GDWOu2023-05-0581449039533149712064.4515003874168475032023-07-06 22:34:11sfgxhqvOLzjdTSNcNaWfEnZqvQraSSuMPazCGhPmSrGuxggqGh111449466287130860562118177510004750271267350957FDhTstrue96247293946402921952995131535667203.3240283E384.473485404447698E307YFdwf2023-02-04294 [...]
+xVJPgVlosBlTYSkmJCqKHMXzbZkNQKInuVMZeYGhsmzUmcLyPx137745493211075991209783701051546835517166168384qcYaifalse8318050110096656524405690917018449922.9617934E371.8901064340036343E307jaKMq2023-05-1275317114043170470995.9654034735914367862023-05-18 08:09:22raGGBnHsNwMZKemkFErUbedNjSllNcKOVUGdTpXcHGSVphHsNE86377304018502081846122308810391870441519757437JCRZStrue1829974183977114228752256792969205767.9090967E371.6286963710372255E308NBHUB2023-05-0 [...]
+dBgFeTKkCfnxCljyGfNEurEzCVgwpsHgmcOfYXiQHxeeQNjQuq1961913761867016982512369059615238191571813320BTfhbfalse652666522281866957533025299230722.1456136E381.2398422714159417E308YOiwg2023-10-2433001899362876139955.7235198795513055732023-06-23 13:46:46jsvmHLHlXCGFKwuqlTwAjdMckElrmqgBWvOuuKuWxcinFZWSky19959088245502706421265289671411088181469730839vUyULtrue952655754382886132164227350822215681.9033253E381.0966562906060974E308XFeKf2023-09-1731084 [...]
+obtYzIHOTKsABVtirEKEMYUYobsYlDJcFbpQUYvGxCcKlnswEG8096984004544201585383739017658796661353001394xchcntrue853141253976762312923177914159380482.8480754E381.055208146200822E308MSkTD2023-11-2420361788179232141281.9718823433892185262023-10-25 11:47:50gdCWZMGESyarjQPopBhDwKnOyDvaUDgQOEDRCmfUAagfnDDPqV8473436731118772451890654127233667151574025969ewJzLtrue6321769209768782446484076920790579202.7134378E381.1883616449174808E308STvOu2023-10-082179 [...]
\ No newline at end of file
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java
index c0cc2d77c2..213771b4e9 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java
@@ -324,6 +324,22 @@ public class LocalFileIT extends TestSuiteBase {
"-c",
"mkdir -p /seatunnel/read/markdown && printf '# E2E Markdown RAG\\n' > /seatunnel/read/markdown/e2e.md");
+ ContainerUtil.copyFileIntoContainers(
+ "/text/e2e.txt", "/seatunnel/read/recursive/e2e.txt", container);
+
+ ContainerUtil.copyFileIntoContainers(
+ "/text/e2e.txt", "/seatunnel/read/recursive/subdir/e2e.txt", container);
+
+ ContainerUtil.copyFileIntoContainers(
+ "/text/e2e.txt",
+ "/seatunnel/read/recursive/subdir/deeper/e2e.txt",
+ container);
+
+ ContainerUtil.copyFileIntoContainers(
+ "/text/e2e.txt",
+ "/seatunnel/read/recursive/subdir/deeper/final/e2e.txt",
+ container);
+
container.execInContainer("mkdir", "-p", "/tmp/fake_empty");
};
@@ -430,6 +446,10 @@ public class LocalFileIT extends TestSuiteBase {
helper.execute("/excel/local_excel_multi_zip_to_assert.conf");
helper.execute("/excel/local_excel_xls_gz_to_assert.conf");
helper.execute("/excel/local_excel_xlsx_gz_to_assert.conf");
+
+ // test read recursive file path
+ helper.execute("/text/local_file_text_recursive_to_assert.conf");
+ helper.execute("/text/local_file_text_non_recursive_to_assert.conf");
}
@TestTemplate
@@ -560,6 +580,108 @@ public class LocalFileIT extends TestSuiteBase {
baseContainer.execInContainer("sh", "-c", "rm -rf /tmp/seatunnel/continuous");
}
+ @TestTemplate
+ @DisabledOnContainer(
+ value = {},
+ type = {EngineType.FLINK, EngineType.SPARK},
+ disabledReason =
+ "Continuous discovery is a long-running job. Local filesystem is not shared between engine master/workers in Flink/Spark E2E.")
+ public void testLocalFileBinaryUpdateModeContinuousDiscoveryWithNonRecursiveScan(
+ TestContainer container) throws IOException, InterruptedException {
+ resetContinuousTestPath();
+
+ String jobId = String.valueOf(JobIdGenerator.newJobId());
+ CompletableFuture<Container.ExecResult> jobFuture =
+ CompletableFuture.supplyAsync(
+ () -> {
+ try {
+ return container.executeJob(
+ "/binary/local_file_binary_update_distcp_continuous_non_recursive.conf",
+ jobId);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ putLocalFile("/tmp/seatunnel/continuous/src/root.bin", "root");
+ putLocalFile("/tmp/seatunnel/continuous/src/subdir/nested.bin", "nested");
+
+ Awaitility.await()
+ .atMost(60, TimeUnit.SECONDS)
+ .untilAsserted(
+ () ->
+ Assertions.assertEquals(
+ "root",
+ readLocalFile("/tmp/seatunnel/continuous/dst/root.bin")));
+
+ Thread.sleep(3000);
+ Assertions.assertFalse(
+ isLocalFileExists("/tmp/seatunnel/continuous/dst/subdir/nested.bin"));
+
+ Container.ExecResult cancelResult = container.cancelJob(jobId);
+ Assertions.assertEquals(0, cancelResult.getExitCode(), cancelResult.getStderr());
+
+ Container.ExecResult execResult;
+ try {
+ execResult = jobFuture.get(120, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new RuntimeException("Wait continuous job exit failed.", e);
+ }
+ Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr());
+
+ baseContainer.execInContainer("sh", "-c", "rm -rf /tmp/seatunnel/continuous");
+ }
+
+ @TestTemplate
+ @DisabledOnContainer(
+ value = {},
+ type = {EngineType.FLINK, EngineType.SPARK},
+ disabledReason =
+ "sync_mode=update needs to compare source/target on the same filesystem. Local filesystem is not shared between engine master/workers in Flink/Spark E2E.")
+ public void testLocalFileBinaryUpdateModeDistcpWithNonRecursiveScan(TestContainer container)
+ throws IOException, InterruptedException {
+ resetUpdateTestPath();
+ putLocalFile("/tmp/seatunnel/update/src/root.bin", "root-updated-v2");
+ putLocalFile("/tmp/seatunnel/update/src/subdir/nested.bin", "nest-updated-v2");
+ putLocalFile("/tmp/seatunnel/update/dst/root.bin", "root-stale-v1");
+ putLocalFile("/tmp/seatunnel/update/dst/subdir/nested.bin", "nest-stale-v1");
+
+ TestHelper helper = new TestHelper(container);
+ helper.execute("/binary/local_file_binary_update_non_recursive_distcp.conf");
+
+ Assertions.assertEquals(
+ "root-updated-v2", readLocalFile("/tmp/seatunnel/update/dst/root.bin"));
+ Assertions.assertEquals(
+ "nest-stale-v1", readLocalFile("/tmp/seatunnel/update/dst/subdir/nested.bin"));
+
+ baseContainer.execInContainer("sh", "-c", "rm -rf /tmp/seatunnel/update");
+ }
+
+ @TestTemplate
+ @DisabledOnContainer(
+ value = {},
+ type = {EngineType.FLINK, EngineType.SPARK},
+ disabledReason =
+ "sync_mode=update needs to compare source/target on the same filesystem. Local filesystem is not shared between engine master/workers in Flink/Spark E2E.")
+ public void testLocalFileBinaryUpdateModeStrictChecksumSkipsNestedChangesWithNonRecursiveScan(
+ TestContainer container) throws IOException, InterruptedException {
+ resetUpdateTestPath();
+ putLocalFile("/tmp/seatunnel/update/src/root.bin", "root-same-v1");
+ putLocalFile("/tmp/seatunnel/update/src/subdir/nested.bin", "nest-new-v1");
+ putLocalFile("/tmp/seatunnel/update/dst/root.bin", "root-same-v1");
+ putLocalFile("/tmp/seatunnel/update/dst/subdir/nested.bin", "nest-old-v1");
+
+ TestHelper helper = new TestHelper(container);
+ helper.execute("/binary/local_file_binary_update_non_recursive_strict_checksum.conf");
+
+ Assertions.assertEquals(
+ "root-same-v1", readLocalFile("/tmp/seatunnel/update/dst/root.bin"));
+ Assertions.assertEquals(
+ "nest-old-v1", readLocalFile("/tmp/seatunnel/update/dst/subdir/nested.bin"));
+
+ baseContainer.execInContainer("sh", "-c", "rm -rf /tmp/seatunnel/update");
+ }
+
@TestTemplate
@DisabledOnContainer(
value = {TestContainerId.SPARK_2_4},
@@ -666,6 +788,12 @@ public class LocalFileIT extends TestSuiteBase {
return result.getStdout() == null ? "" : result.getStdout().trim();
}
+ private boolean isLocalFileExists(String filePath) throws IOException, InterruptedException {
+ Container.ExecResult result =
+ baseContainer.execInContainer("sh", "-c", "test -f '" + filePath + "'");
+ return result.getExitCode() == 0;
+ }
+
private long getLocalFileMtimeSeconds(String filePath)
throws IOException, InterruptedException {
Container.ExecResult result =
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_distcp_continuous_non_recursive.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_distcp_continuous_non_recursive.conf
new file mode 100644
index 0000000000..2a90543b72
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_distcp_continuous_non_recursive.conf
@@ -0,0 +1,47 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "STREAMING"
+ checkpoint.interval = 5000
+}
+
+source {
+ LocalFile {
+ path = "/tmp/seatunnel/continuous/src"
+ file_format_type = "binary"
+
+ discovery_mode = "continuous"
+ scan_interval = "1S"
+ start_mode = "earliest"
+
+ sync_mode = "update"
+ target_path = "/tmp/seatunnel/continuous/dst"
+ update_strategy = "distcp"
+ compare_mode = "len_mtime"
+ recursive_file_scan = false
+ }
+}
+
+sink {
+ LocalFile {
+ path = "/tmp/seatunnel/continuous/dst"
+ tmp_path = "/tmp/seatunnel/continuous/tmp"
+ file_format_type = "binary"
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_non_recursive_distcp.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_non_recursive_distcp.conf
new file mode 100644
index 0000000000..860e4dfeb6
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_non_recursive_distcp.conf
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+}
+
+source {
+ LocalFile {
+ path = "/tmp/seatunnel/update/src"
+ file_format_type = "binary"
+
+ sync_mode = "update"
+ target_path = "/tmp/seatunnel/update/dst"
+ update_strategy = "distcp"
+ compare_mode = "len_mtime"
+ recursive_file_scan = false
+ }
+}
+
+sink {
+ LocalFile {
+ path = "/tmp/seatunnel/update/dst"
+ tmp_path = "/tmp/seatunnel/update/tmp"
+ file_format_type = "binary"
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_non_recursive_strict_checksum.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_non_recursive_strict_checksum.conf
new file mode 100644
index 0000000000..fe53193e10
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/binary/local_file_binary_update_non_recursive_strict_checksum.conf
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+}
+
+source {
+ LocalFile {
+ path = "/tmp/seatunnel/update/src"
+ file_format_type = "binary"
+
+ sync_mode = "update"
+ target_path = "/tmp/seatunnel/update/dst"
+ update_strategy = "strict"
+ compare_mode = "checksum"
+ recursive_file_scan = false
+ }
+}
+
+sink {
+ LocalFile {
+ path = "/tmp/seatunnel/update/dst"
+ tmp_path = "/tmp/seatunnel/update/tmp"
+ file_format_type = "binary"
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/local_file_text_non_recursive_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/local_file_text_non_recursive_to_assert.conf
new file mode 100644
index 0000000000..36419729ea
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/local_file_text_non_recursive_to_assert.conf
@@ -0,0 +1,117 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+
+ # You can set spark configuration here
+ spark.app.name = "SeaTunnel"
+ spark.executor.instances = 2
+ spark.executor.cores = 1
+ spark.executor.memory = "1g"
+ spark.master = local
+}
+
+source {
+ LocalFile {
+ path = "/seatunnel/read/recursive"
+ schema = {
+ fields {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ c_row = {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ }
+ }
+ }
+ file_format_type = "text"
+ recursive_file_scan = false
+ plugin_output = "fake"
+ }
+}
+
+sink {
+ Assert {
+ rules {
+ row_rules = [
+ {
+ rule_type = MAX_ROW
+ rule_value = 5
+ },
+ {
+ rule_type = MIN_ROW
+ rule_value = 5
+ }
+ ],
+ field_rules = [
+ {
+ field_name = c_string
+ field_type = string
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = c_boolean
+ field_type = boolean
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = c_double
+ field_type = double
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ }
+ ]
+ }
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/local_file_text_recursive_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/local_file_text_recursive_to_assert.conf
new file mode 100644
index 0000000000..19d3a8a216
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/text/local_file_text_recursive_to_assert.conf
@@ -0,0 +1,117 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+
+ # You can set spark configuration here
+ spark.app.name = "SeaTunnel"
+ spark.executor.instances = 2
+ spark.executor.cores = 1
+ spark.executor.memory = "1g"
+ spark.master = local
+}
+
+source {
+ LocalFile {
+ path = "/seatunnel/read/recursive"
+ schema = {
+ fields {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ c_row = {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ }
+ }
+ }
+ file_format_type = "text"
+ recursive_file_scan = true
+ plugin_output = "fake"
+ }
+}
+
+sink {
+ Assert {
+ rules {
+ row_rules = [
+ {
+ rule_type = MAX_ROW
+ rule_value = 20
+ },
+ {
+ rule_type = MIN_ROW
+ rule_value = 20
+ }
+ ],
+ field_rules = [
+ {
+ field_name = c_string
+ field_type = string
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = c_boolean
+ field_type = boolean
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = c_double
+ field_type = double
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ }
+ ]
+ }
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java
index c11282f140..658cfaab00 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java
@@ -125,6 +125,27 @@ public class SftpFileIT extends TestSuiteBase implements TestResource {
"/text/e2e.txt",
"/home/seatunnel/tmp/seatunnel/read/wildcard/e2e.txt",
sftpContainer);
+
+ ContainerUtil.copyFileIntoContainers(
+ "/text/e2e.txt",
+ "/home/seatunnel/tmp/seatunnel/read/recursive/e2e.txt",
+ sftpContainer);
+
+ ContainerUtil.copyFileIntoContainers(
+ "/text/e2e.txt",
+ "/home/seatunnel/tmp/seatunnel/read/recursive/subdir/e2e.txt",
+ sftpContainer);
+
+ ContainerUtil.copyFileIntoContainers(
+ "/text/e2e.txt",
+ "/home/seatunnel/tmp/seatunnel/read/recursive/subdir/deeper/e2e.txt",
+ sftpContainer);
+
+ ContainerUtil.copyFileIntoContainers(
+ "/text/e2e.txt",
+ "/home/seatunnel/tmp/seatunnel/read/recursive/subdir/deeper/final/e2e.txt",
+ sftpContainer);
+
Container.ExecResult chownResult =
sftpContainer.execInContainer(
"sh", "-c", "chown -R seatunnel /home/seatunnel/tmp/");
@@ -335,6 +356,115 @@ public class SftpFileIT extends TestSuiteBase implements TestResource {
}
}
+ @TestTemplate
+ @DisabledOnContainer(
+ value = {},
+ type = {EngineType.FLINK, EngineType.SPARK},
+ disabledReason = "Continuous discovery is a long-running job; only run in zeta engine.")
+ public void testSftpBinaryUpdateModeContinuousDiscoveryWithNonRecursiveScan(
+ TestContainer container) throws IOException, InterruptedException {
+ resetContinuousTestPath();
+ try {
+ String jobId = String.valueOf(JobIdGenerator.newJobId());
+ CompletableFuture<Container.ExecResult> jobFuture =
+ CompletableFuture.supplyAsync(
+ () -> {
+ try {
+ return container.executeJob(
+ "/text/sftp_binary_update_distcp_continuous_non_recursive.conf",
+ jobId);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ putSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/continuous/src/root.bin", "root");
+ putSftpFile(
+ SFTP_CONTAINER_HOME + "/tmp/seatunnel/continuous/src/subdir/nested.bin",
+ "nested");
+
+ Awaitility.await()
+ .atMost(120, TimeUnit.SECONDS)
+ .pollInterval(2, TimeUnit.SECONDS)
+ .untilAsserted(
+ () ->
+ Assertions.assertEquals(
+ "root",
+ readSftpFile(
+ SFTP_CONTAINER_HOME
+ + "/tmp/seatunnel/continuous/dst/root.bin")));
+
+ Thread.sleep(3000);
+ Assertions.assertFalse(
+ isSftpFileExists(
+ SFTP_CONTAINER_HOME
+ + "/tmp/seatunnel/continuous/dst/subdir/nested.bin"));
+
+ Container.ExecResult cancelResult = container.cancelJob(jobId);
+ Assertions.assertEquals(0, cancelResult.getExitCode(), cancelResult.getStderr());
+
+ Container.ExecResult execResult;
+ try {
+ execResult = jobFuture.get(120, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new RuntimeException("Wait continuous job exit failed.", e);
+ }
+ Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr());
+ } finally {
+ deleteFileFromContainer(SFTP_CONTAINER_HOME + "/tmp/seatunnel/continuous");
+ }
+ }
+
+ @TestTemplate
+ public void testSftpBinaryUpdateModeDistcpWithNonRecursiveScan(TestContainer container)
+ throws IOException, InterruptedException {
+ resetUpdateTestPath();
+ putSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/src/root.bin", "root-updated-v2");
+ putSftpFile(
+ SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/src/subdir/nested.bin",
+ "nest-updated-v2");
+ putSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/dst/root.bin", "root-stale-v1");
+ putSftpFile(
+ SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/dst/subdir/nested.bin",
+ "nest-stale-v1");
+
+ TestHelper helper = new TestHelper(container);
+ helper.execute("/text/sftp_binary_update_non_recursive_distcp.conf");
+
+ Assertions.assertEquals(
+ "root-updated-v2",
+ readSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/dst/root.bin"));
+ Assertions.assertEquals(
+ "nest-stale-v1",
+ readSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/dst/subdir/nested.bin"));
+
+ deleteFileFromContainer(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update");
+ }
+
+ @TestTemplate
+ public void testSftpBinaryUpdateModeStrictChecksumSkipsNestedChangesWithNonRecursiveScan(
+ TestContainer container) throws IOException, InterruptedException {
+ resetUpdateTestPath();
+ putSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/src/root.bin", "root-same-v1");
+ putSftpFile(
+ SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/src/subdir/nested.bin", "nest-new-v1");
+ putSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/dst/root.bin", "root-same-v1");
+ putSftpFile(
+ SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/dst/subdir/nested.bin", "nest-old-v1");
+
+ TestHelper helper = new TestHelper(container);
+ helper.execute("/text/sftp_binary_update_non_recursive_strict_checksum.conf");
+
+ Assertions.assertEquals(
+ "root-same-v1",
+ readSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/dst/root.bin"));
+ Assertions.assertEquals(
+ "nest-old-v1",
+ readSftpFile(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update/dst/subdir/nested.bin"));
+
+ deleteFileFromContainer(SFTP_CONTAINER_HOME + "/tmp/seatunnel/update");
+ }
+
@TestTemplate
public void testMultipleTableAndSaveMode(TestContainer container)
throws IOException, InterruptedException {
@@ -446,6 +576,13 @@ public class SftpFileIT extends TestSuiteBase implements TestResource {
return catResult.getStdout() == null ? "" : catResult.getStdout().trim();
}
+ private boolean isSftpFileExists(String containerPath)
+ throws IOException, InterruptedException {
+ Container.ExecResult result =
+ sftpContainer.execInContainer("sh", "-c", "test -f '" + containerPath + "'");
+ return result.getExitCode() == 0;
+ }
+
private void waitUntilContainerTimeAfter(long epochSeconds) {
Awaitility.await()
.atMost(10, TimeUnit.SECONDS)
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_distcp_continuous_non_recursive.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_distcp_continuous_non_recursive.conf
new file mode 100644
index 0000000000..de3f99a7f7
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_distcp_continuous_non_recursive.conf
@@ -0,0 +1,64 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "STREAMING"
+ checkpoint.interval = 5000
+
+ # You can set spark configuration here
+ spark.app.name = "SeaTunnel"
+ spark.executor.instances = 1
+ spark.executor.cores = 1
+ spark.executor.memory = "1g"
+ spark.master = local
+}
+
+source {
+ SftpFile {
+ host = "sftp"
+ port = 22
+ user = seatunnel
+ password = pass
+
+ path = "tmp/seatunnel/continuous/src"
+ file_format_type = "binary"
+
+ discovery_mode = "continuous"
+ scan_interval = "1S"
+ start_mode = "earliest"
+
+ sync_mode = "update"
+ target_path = "tmp/seatunnel/continuous/dst"
+ update_strategy = "distcp"
+ compare_mode = "len_mtime"
+ recursive_file_scan = false
+ }
+}
+
+sink {
+ SftpFile {
+ host = "sftp"
+ port = 22
+ user = seatunnel
+ password = pass
+
+ path = "tmp/seatunnel/continuous/dst"
+ tmp_path = "tmp/seatunnel/continuous/tmp"
+ file_format_type = "binary"
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_non_recursive_distcp.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_non_recursive_distcp.conf
new file mode 100644
index 0000000000..babebe240e
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_non_recursive_distcp.conf
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+
+ # You can set spark configuration here
+ spark.app.name = "SeaTunnel"
+ spark.executor.instances = 1
+ spark.executor.cores = 1
+ spark.executor.memory = "1g"
+ spark.master = local
+}
+
+source {
+ SftpFile {
+ host = "sftp"
+ port = 22
+ user = seatunnel
+ password = pass
+
+ path = "tmp/seatunnel/update/src"
+ file_format_type = "binary"
+
+ sync_mode = "update"
+ target_path = "tmp/seatunnel/update/dst"
+ update_strategy = "distcp"
+ compare_mode = "len_mtime"
+ recursive_file_scan = false
+ }
+}
+
+sink {
+ SftpFile {
+ host = "sftp"
+ port = 22
+ user = seatunnel
+ password = pass
+
+ path = "tmp/seatunnel/update/dst"
+ tmp_path = "tmp/seatunnel/update/tmp"
+ file_format_type = "binary"
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_non_recursive_strict_checksum.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_non_recursive_strict_checksum.conf
new file mode 100644
index 0000000000..34ef8272cd
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_binary_update_non_recursive_strict_checksum.conf
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+
+ # You can set spark configuration here
+ spark.app.name = "SeaTunnel"
+ spark.executor.instances = 1
+ spark.executor.cores = 1
+ spark.executor.memory = "1g"
+ spark.master = local
+}
+
+source {
+ SftpFile {
+ host = "sftp"
+ port = 22
+ user = seatunnel
+ password = pass
+
+ path = "tmp/seatunnel/update/src"
+ file_format_type = "binary"
+
+ sync_mode = "update"
+ target_path = "tmp/seatunnel/update/dst"
+ update_strategy = "strict"
+ compare_mode = "checksum"
+ recursive_file_scan = false
+ }
+}
+
+sink {
+ SftpFile {
+ host = "sftp"
+ port = 22
+ user = seatunnel
+ password = pass
+
+ path = "tmp/seatunnel/update/dst"
+ tmp_path = "tmp/seatunnel/update/tmp"
+ file_format_type = "binary"
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_file_text_non_recursive_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_file_text_non_recursive_to_assert.conf
new file mode 100644
index 0000000000..42b7bbc30f
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_file_text_non_recursive_to_assert.conf
@@ -0,0 +1,122 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+
+ # You can set spark configuration here
+ spark.app.name = "SeaTunnel"
+ spark.executor.instances = 1
+ spark.executor.cores = 1
+ spark.executor.memory = "1g"
+ spark.master = local
+}
+
+source {
+ SftpFile {
+ host = "sftp"
+ port = 22
+ user = seatunnel
+ password = pass
+ path = "tmp/seatunnel/read/recursive"
+ file_format_type = "text"
+ recursive_file_scan = false
+ plugin_output = "sftp"
+ schema = {
+ fields {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ c_row = {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ }
+ }
+ }
+ }
+}
+
+sink {
+ Assert {
+ plugin_input = "sftp"
+ rules {
+ row_rules = [
+ {
+ rule_type = MAX_ROW
+ rule_value = 5
+ },
+ {
+ rule_type = MIN_ROW
+ rule_value = 5
+ }
+ ],
+ field_rules = [
+ {
+ field_name = c_string
+ field_type = string
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = c_boolean
+ field_type = boolean
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = c_double
+ field_type = double
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ }
+ ]
+ }
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_file_text_recursive_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_file_text_recursive_to_assert.conf
new file mode 100644
index 0000000000..f81e4625f4
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_file_text_recursive_to_assert.conf
@@ -0,0 +1,122 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+
+ # You can set spark configuration here
+ spark.app.name = "SeaTunnel"
+ spark.executor.instances = 1
+ spark.executor.cores = 1
+ spark.executor.memory = "1g"
+ spark.master = local
+}
+
+source {
+ SftpFile {
+ host = "sftp"
+ port = 22
+ user = seatunnel
+ password = pass
+ path = "tmp/seatunnel/read/recursive"
+ file_format_type = "text"
+ recursive_file_scan = true
+ plugin_output = "sftp"
+ schema = {
+ fields {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ c_row = {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ }
+ }
+ }
+ }
+}
+
+sink {
+ Assert {
+ plugin_input = "sftp"
+ rules {
+ row_rules = [
+ {
+ rule_type = MAX_ROW
+ rule_value = 20
+ },
+ {
+ rule_type = MIN_ROW
+ rule_value = 20
+ }
+ ],
+ field_rules = [
+ {
+ field_name = c_string
+ field_type = string
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = c_boolean
+ field_type = boolean
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = c_double
+ field_type = double
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ }
+ ]
+ }
+ }
+}