This is an automated email from the ASF dual-hosted git repository.
gaojun2048 pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 1a48a9c493 [Improve][Files] Support write fixed/timestamp as int96 of parquet (#6971)
1a48a9c493 is described below
commit 1a48a9c493a24364fd54e34d8d2447b593957299
Author: hailin0 <wa...@apache.org>
AuthorDate: Thu Jun 13 10:52:51 2024 +0800
[Improve][Files] Support write fixed/timestamp as int96 of parquet (#6971)
---
docs/en/connector-v2/sink/CosFile.md | 68 ++++++----
docs/en/connector-v2/sink/FtpFile.md | 70 +++++-----
docs/en/connector-v2/sink/HdfsFile.md | 64 ++++-----
docs/en/connector-v2/sink/LocalFile.md | 62 +++++----
docs/en/connector-v2/sink/OssFile.md | 68 ++++++----
docs/en/connector-v2/sink/OssJindoFile.md | 68 ++++++----
docs/en/connector-v2/sink/S3File.md | 76 ++++++-----
docs/en/connector-v2/sink/SftpFile.md | 68 ++++++----
.../seatunnel/file/config/BaseSinkConfig.java | 15 +++
.../seatunnel/file/config/HadoopConf.java | 2 -
.../seatunnel/file/sink/config/FileSinkConfig.java | 21 +++
.../file/sink/writer/ParquetWriteStrategy.java | 83 +++++++++++-
.../file/writer/ParquetWriteStrategyTest.java | 150 +++++++++++++++++++++
.../file/cos/sink/CosFileSinkFactory.java | 4 +-
.../file/ftp/sink/FtpFileSinkFactory.java | 4 +-
.../file/hdfs/sink/HdfsFileSinkFactory.java | 4 +-
.../file/oss/jindo/sink/OssFileSinkFactory.java | 4 +-
.../file/local/sink/LocalFileSinkFactory.java | 4 +-
.../file/oss/sink/OssFileSinkFactory.java | 4 +-
.../seatunnel/file/s3/sink/S3FileSinkFactory.java | 4 +-
.../file/sftp/sink/SftpFileSinkFactory.java | 4 +-
21 files changed, 595 insertions(+), 252 deletions(-)
diff --git a/docs/en/connector-v2/sink/CosFile.md b/docs/en/connector-v2/sink/CosFile.md
index 4bf812666e..612273a936 100644
--- a/docs/en/connector-v2/sink/CosFile.md
+++ b/docs/en/connector-v2/sink/CosFile.md
@@ -34,35 +34,37 @@ By default, we use 2PC commit to ensure `exactly-once`
## Options
-| Name | Type | Required | Default | Description |
-|----------------------------------|---------|----------|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------|
-| path | string | yes | - | |
-| tmp_path | string | no | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. Need a COS dir. |
-| bucket | string | yes | - | |
-| secret_id | string | yes | - | |
-| secret_key | string | yes | - | |
-| region | string | yes | - | |
-| custom_filename | boolean | no | false | Whether you need custom the filename |
-| file_name_expression | string | no | "${transactionId}" | Only used when custom_filename is true |
-| filename_time_format | string | no | "yyyy.MM.dd" | Only used when custom_filename is true |
-| file_format_type | string | no | "csv" | |
-| field_delimiter | string | no | '\001' | Only used when file_format is text |
-| row_delimiter | string | no | "\n" | Only used when file_format is text |
-| have_partition | boolean | no | false | Whether you need processing partitions. |
-| partition_by | array | no | - | Only used then have_partition is true |
-| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | Only used then have_partition is true |
-| is_partition_field_write_in_file | boolean | no | false | Only used then have_partition is true |
-| sink_columns | array | no | | When this parameter is empty, all fields are sink columns |
-| is_enable_transaction | boolean | no | true | |
-| batch_size | int | no | 1000000 | |
-| compress_codec | string | no | none | |
-| common-options | object | no | - | |
-| max_rows_in_memory | int | no | - | Only used when file_format is excel. |
-| sheet_name | string | no | Sheet${Random number} | Only used when file_format is excel. |
-| xml_root_tag | string | no | RECORDS | Only used when file_format is xml. |
-| xml_row_tag | string | no | RECORD | Only used when file_format is xml. |
-| xml_use_attr_format | boolean | no | - | Only used when file_format is xml. |
-| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
+| Name | Type | Required | Default | Description |
+|---------------------------------------|---------|----------|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------|
+| path | string | yes | - | |
+| tmp_path | string | no | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. Need a COS dir. |
+| bucket | string | yes | - | |
+| secret_id | string | yes | - | |
+| secret_key | string | yes | - | |
+| region | string | yes | - | |
+| custom_filename | boolean | no | false | Whether you need custom the filename |
+| file_name_expression | string | no | "${transactionId}" | Only used when custom_filename is true |
+| filename_time_format | string | no | "yyyy.MM.dd" | Only used when custom_filename is true |
+| file_format_type | string | no | "csv" | |
+| field_delimiter | string | no | '\001' | Only used when file_format is text |
+| row_delimiter | string | no | "\n" | Only used when file_format is text |
+| have_partition | boolean | no | false | Whether you need processing partitions. |
+| partition_by | array | no | - | Only used then have_partition is true |
+| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | Only used then have_partition is true |
+| is_partition_field_write_in_file | boolean | no | false | Only used then have_partition is true |
+| sink_columns | array | no | | When this parameter is empty, all fields are sink columns |
+| is_enable_transaction | boolean | no | true | |
+| batch_size | int | no | 1000000 | |
+| compress_codec | string | no | none | |
+| common-options | object | no | - | |
+| max_rows_in_memory | int | no | - | Only used when file_format is excel. |
+| sheet_name | string | no | Sheet${Random number} | Only used when file_format is excel. |
+| xml_root_tag | string | no | RECORDS | Only used when file_format is xml. |
+| xml_row_tag | string | no | RECORD | Only used when file_format is xml. |
+| xml_use_attr_format | boolean | no | - | Only used when file_format is xml. |
+| parquet_avro_write_timestamp_as_int96 | boolean | no | false | Only used when file_format is parquet. |
+| parquet_avro_write_fixed_as_int96 | array | no | - | Only used when file_format is parquet. |
+| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
### path [string]
@@ -207,6 +209,14 @@ Specifies the tag name of the data rows within the XML file.
Specifies Whether to process data using the tag attribute format.
+### parquet_avro_write_timestamp_as_int96 [boolean]
+
+Support writing Parquet INT96 from a timestamp, only valid for parquet files.
+
+### parquet_avro_write_fixed_as_int96 [array]
+
+Support writing Parquet INT96 from a 12-byte field, only valid for parquet files.
+
### encoding [string]
Only used when file_format_type is json,text,csv,xml.
diff --git a/docs/en/connector-v2/sink/FtpFile.md b/docs/en/connector-v2/sink/FtpFile.md
index 3b429dfe04..01c46190af 100644
--- a/docs/en/connector-v2/sink/FtpFile.md
+++ b/docs/en/connector-v2/sink/FtpFile.md
@@ -32,36 +32,38 @@ By default, we use 2PC commit to ensure `exactly-once`
## Options
-| Name | Type | Required | Default | Description |
-|----------------------------------|---------|----------|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------|
-| host | string | yes | - | |
-| port | int | yes | - | |
-| user | string | yes | - | |
-| password | string | yes | - | |
-| path | string | yes | - | |
-| tmp_path | string | yes | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. Need a FTP dir. |
-| connection_mode | string | no | active_local | The target ftp connection mode |
-| custom_filename | boolean | no | false | Whether you need custom the filename |
-| file_name_expression | string | no | "${transactionId}" | Only used when custom_filename is true |
-| filename_time_format | string | no | "yyyy.MM.dd" | Only used when custom_filename is true |
-| file_format_type | string | no | "csv" | |
-| field_delimiter | string | no | '\001' | Only used when file_format_type is text |
-| row_delimiter | string | no | "\n" | Only used when file_format_type is text |
-| have_partition | boolean | no | false | Whether you need processing partitions. |
-| partition_by | array | no | - | Only used then have_partition is true |
-| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | Only used then have_partition is true |
-| is_partition_field_write_in_file | boolean | no | false | Only used then have_partition is true |
-| sink_columns | array | no | | When this parameter is empty, all fields are sink columns |
-| is_enable_transaction | boolean | no | true | |
-| batch_size | int | no | 1000000 | |
-| compress_codec | string | no | none | |
-| common-options | object | no | - | |
-| max_rows_in_memory | int | no | - | Only used when file_format_type is excel. |
-| sheet_name | string | no | Sheet${Random number} | Only used when file_format_type is excel. |
-| xml_root_tag | string | no | RECORDS | Only used when file_format is xml. |
-| xml_row_tag | string | no | RECORD | Only used when file_format is xml. |
-| xml_use_attr_format | boolean | no | - | Only used when file_format is xml. |
-| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
+| Name | Type | Required | Default | Description |
+|---------------------------------------|---------|----------|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------|
+| host | string | yes | - | |
+| port | int | yes | - | |
+| user | string | yes | - | |
+| password | string | yes | - | |
+| path | string | yes | - | |
+| tmp_path | string | yes | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. Need a FTP dir. |
+| connection_mode | string | no | active_local | The target ftp connection mode |
+| custom_filename | boolean | no | false | Whether you need custom the filename |
+| file_name_expression | string | no | "${transactionId}" | Only used when custom_filename is true |
+| filename_time_format | string | no | "yyyy.MM.dd" | Only used when custom_filename is true |
+| file_format_type | string | no | "csv" | |
+| field_delimiter | string | no | '\001' | Only used when file_format_type is text |
+| row_delimiter | string | no | "\n" | Only used when file_format_type is text |
+| have_partition | boolean | no | false | Whether you need processing partitions. |
+| partition_by | array | no | - | Only used then have_partition is true |
+| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | Only used then have_partition is true |
+| is_partition_field_write_in_file | boolean | no | false | Only used then have_partition is true |
+| sink_columns | array | no | | When this parameter is empty, all fields are sink columns |
+| is_enable_transaction | boolean | no | true | |
+| batch_size | int | no | 1000000 | |
+| compress_codec | string | no | none | |
+| common-options | object | no | - | |
+| max_rows_in_memory | int | no | - | Only used when file_format_type is excel. |
+| sheet_name | string | no | Sheet${Random number} | Only used when file_format_type is excel. |
+| xml_root_tag | string | no | RECORDS | Only used when file_format is xml. |
+| xml_row_tag | string | no | RECORD | Only used when file_format is xml. |
+| xml_use_attr_format | boolean | no | - | Only used when file_format is xml. |
+| parquet_avro_write_timestamp_as_int96 | boolean | no | false | Only used when file_format is parquet. |
+| parquet_avro_write_fixed_as_int96 | array | no | - | Only used when file_format is parquet. |
+| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
### host [string]
@@ -212,6 +214,14 @@ Specifies the tag name of the data rows within the XML file.
Specifies Whether to process data using the tag attribute format.
+### parquet_avro_write_timestamp_as_int96 [boolean]
+
+Support writing Parquet INT96 from a timestamp, only valid for parquet files.
+
+### parquet_avro_write_fixed_as_int96 [array]
+
+Support writing Parquet INT96 from a 12-byte field, only valid for parquet files.
+
### encoding [string]
Only used when file_format_type is json,text,csv,xml.
diff --git a/docs/en/connector-v2/sink/HdfsFile.md b/docs/en/connector-v2/sink/HdfsFile.md
index d138089b2e..d42aa98967 100644
--- a/docs/en/connector-v2/sink/HdfsFile.md
+++ b/docs/en/connector-v2/sink/HdfsFile.md
@@ -38,37 +38,39 @@ Output data to hdfs file
## Sink Options
-| Name | Type | Required | Default | Description [...]
-|----------------------------------|---------|----------|--------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ [...]
-| fs.defaultFS | string | yes | - | The hadoop cluster address that start with `hdfs://`, for example: `hdfs://hadoopcluster` [...]
-| path | string | yes | - | The target dir path is required. [...]
-| tmp_path | string | yes | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. Need a hdfs path. [...]
-| hdfs_site_path | string | no | - | The path of `hdfs-site.xml`, used to load ha configuration of namenodes [...]
-| custom_filename | boolean | no | false | Whether you need custom the filename [...]
-| file_name_expression | string | no | "${transactionId}" | Only used when `custom_filename` is `true`.`file_name_expression` describes the file expression which will be created into the `path`. We can add the variable `${now}` or `${uuid}` in the `file_name_expression`, like `test_${uuid}_${now}`,`${now}` represents the current time, and its format can be defined by specifying the option `filename_time_format`.Please note that, If `is_enable_tr [...]
-| filename_time_format | string | no | "yyyy.MM.dd" | Only used when `custom_filename` is `true`.When the format in the `file_name_expression` parameter is `xxxx-${now}` , `filename_time_format` can specify the time format of the path, and the default value is `yyyy.MM.dd` . The commonly used time formats are listed as follows:[y:Year,M:Month,d:Day of month,H:Hour in day (0-23),m:Minute in hour,s:Second in minute] [...]
-| file_format_type | string | no | "csv" | We supported as the following file types:`text` `csv` `parquet` `orc` `json` `excel` `xml` `binary`.Please note that, The final file name will end with the file_format's suffix, the suffix of the text file is `txt`. [...]
-| field_delimiter | string | no | '\001' | Only used when file_format is text,The separator between columns in a row of data. Only needed by `text` file format. [...]
-| row_delimiter | string | no | "\n" | Only used when file_format is text,The separator between rows in a file. Only needed by `text` file format. [...]
-| have_partition | boolean | no | false | Whether you need processing partitions. [...]
-| partition_by | array | no | - | Only used then have_partition is true,Partition data based on selected fields. [...]
-| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | Only used then have_partition is true,If the `partition_by` is specified, we will generate the corresponding partition directory based on the partition information, and the final file will be placed in the partition directory. Default `partition_dir_expression` is `${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/`. `k0` is the first partition field and `v0` is the value of the first partition f [...]
-| is_partition_field_write_in_file | boolean | no | false | Only used when `have_partition` is `true`. If `is_partition_field_write_in_file` is `true`, the partition field and the value of it will be write into data file.For example, if you want to write a Hive Data File, Its value should be `false`. [...]
-| sink_columns | array | no | | When this parameter is empty, all fields are sink columns.Which columns need be write to file, default value is all of the columns get from `Transform` or `Source`. The order of the fields determines the order in which the file is actually written. [...]
-| is_enable_transaction | boolean | no | true | If `is_enable_transaction` is true, we will ensure that data will not be lost or duplicated when it is written to the target directory.Please note that, If `is_enable_transaction` is `true`, we will auto add `${transactionId}_` in the head of the file.Only support `true` now. [...]
-| batch_size | int | no | 1000000 | The maximum number of rows in a file. For SeaTunnel Engine, the number of lines in the file is determined by `batch_size` and `checkpoint.interval` jointly decide. If the value of `checkpoint.interval` is large enough, sink writer will write rows in a file until the rows in the file larger than `batch_size`. If `checkpoint.interval` is small, the sink writer will create a new file when [...]
-| compress_codec | string | no | none | The compress codec of files and the details that supported as the following shown:[txt: `lzo` `none`,json: `lzo` `none`,csv: `lzo` `none`,orc: `lzo` `snappy` `lz4` `zlib` `none`,parquet: `lzo` `snappy` `lz4` `gzip` `brotli` `zstd` `none`].Tips: excel type does not support any compression format. [...]
-| krb5_path | string | no | /etc/krb5.conf | The krb5 path of kerberos [...]
-| kerberos_principal | string | no | - | The principal of kerberos [...]
-| kerberos_keytab_path | string | no | - | The keytab path of kerberos [...]
-| compress_codec | string | no | none | compress codec [...]
-| common-options | object | no | - | Sink plugin common parameters, please refer to [Sink Common Options](common-options.md) for details [...]
-| max_rows_in_memory | int | no | - | Only used when file_format is excel.When File Format is Excel,The maximum number of data items that can be cached in the memory. [...]
-| sheet_name | string | no | Sheet${Random number} | Only used when file_format is excel.Writer the sheet of the workbook [...]
-| xml_root_tag | string | no | RECORDS | Only used when file_format is xml, specifies the tag name of the root element within the XML file. [...]
-| xml_row_tag | string | no | RECORD | Only used when file_format is xml, specifies the tag name of the data rows within the XML file [...]
-| xml_use_attr_format | boolean | no | - | Only used when file_format is xml, specifies Whether to process data using the tag attribute format. [...]
-| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. [...]
+| Name | Type | Required | Default | Description [...]
+|---------------------------------------|---------|----------|--------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [...]
+| fs.defaultFS | string | yes | - | The hadoop cluster address that start with `hdfs://`, for example: `hdfs://hadoopcluster` [...]
+| path | string | yes | - | The target dir path is required. [...]
+| tmp_path | string | yes | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. Need a hdfs path. [...]
+| hdfs_site_path | string | no | - | The path of `hdfs-site.xml`, used to load ha configuration of namenodes [...]
+| custom_filename | boolean | no | false | Whether you need custom the filename [...]
+| file_name_expression | string | no | "${transactionId}" | Only used when `custom_filename` is `true`.`file_name_expression` describes the file expression which will be created into the `path`. We can add the variable `${now}` or `${uuid}` in the `file_name_expression`, like `test_${uuid}_${now}`,`${now}` represents the current time, and its format can be defined by specifying the option `filename_time_format`.Please note that, If `is_enab [...]
+| filename_time_format | string | no | "yyyy.MM.dd" | Only used when `custom_filename` is `true`.When the format in the `file_name_expression` parameter is `xxxx-${now}` , `filename_time_format` can specify the time format of the path, and the default value is `yyyy.MM.dd` . The commonly used time formats are listed as follows:[y:Year,M:Month,d:Day of month,H:Hour in day (0-23),m:Minute in hour,s:Second in minute] [...]
+| file_format_type | string | no | "csv" | We supported as the following file types:`text` `csv` `parquet` `orc` `json` `excel` `xml` `binary`.Please note that, The final file name will end with the file_format's suffix, the suffix of the text file is `txt`. [...]
+| field_delimiter | string | no | '\001' | Only used when file_format is text,The separator between columns in a row of data. Only needed by `text` file format. [...]
+| row_delimiter | string | no | "\n" | Only used when file_format is text,The separator between rows in a file. Only needed by `text` file format. [...]
+| have_partition | boolean | no | false | Whether you need processing partitions. [...]
+| partition_by | array | no | - | Only used then have_partition is true,Partition data based on selected fields. [...]
+| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | Only used then have_partition is true,If the `partition_by` is specified, we will generate the corresponding partition directory based on the partition information, and the final file will be placed in the partition directory. Default `partition_dir_expression` is `${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/`. `k0` is the first partition field and `v0` is the value of the first partit [...]
+| is_partition_field_write_in_file | boolean | no | false | Only used when `have_partition` is `true`. If `is_partition_field_write_in_file` is `true`, the partition field and the value of it will be write into data file.For example, if you want to write a Hive Data File, Its value should be `false`. [...]
+| sink_columns | array | no | | When this parameter is empty, all fields are sink columns.Which columns need be write to file, default value is all of the columns get from `Transform` or `Source`. The order of the fields determines the order in which the file is actually written. [...]
+| is_enable_transaction | boolean | no | true | If `is_enable_transaction` is true, we will ensure that data will not be lost or duplicated when it is written to the target directory.Please note that, If `is_enable_transaction` is `true`, we will auto add `${transactionId}_` in the head of the file.Only support `true` now. [...]
+| batch_size | int | no | 1000000 | The maximum number of rows in a file. For SeaTunnel Engine, the number of lines in the file is determined by `batch_size` and `checkpoint.interval` jointly decide. If the value of `checkpoint.interval` is large enough, sink writer will write rows in a file until the rows in the file larger than `batch_size`. If `checkpoint.interval` is small, the sink writer will create a new file [...]
+| compress_codec | string | no | none | The compress codec of files and the details that supported as the following shown:[txt: `lzo` `none`,json: `lzo` `none`,csv: `lzo` `none`,orc: `lzo` `snappy` `lz4` `zlib` `none`,parquet: `lzo` `snappy` `lz4` `gzip` `brotli` `zstd` `none`].Tips: excel type does not support any compression format. [...]
+| krb5_path | string | no | /etc/krb5.conf | The krb5 path of kerberos [...]
+| kerberos_principal | string | no | - | The principal of kerberos [...]
+| kerberos_keytab_path | string | no | - | The keytab path of kerberos [...]
+| compress_codec | string | no | none | compress codec [...]
+| common-options | object | no | - | Sink plugin common parameters, please refer to [Sink Common Options](common-options.md) for details [...]
+| max_rows_in_memory | int | no | - | Only used when file_format is excel.When File Format is Excel,The maximum number of data items that can be cached in the memory. [...]
+| sheet_name | string | no | Sheet${Random number} | Only used when file_format is excel.Writer the sheet of the workbook [...]
+| xml_root_tag | string | no | RECORDS | Only used when file_format is xml, specifies the tag name of the root element within the XML file. [...]
+| xml_row_tag | string | no | RECORD | Only used when file_format is xml, specifies the tag name of the data rows within the XML file [...]
+| xml_use_attr_format | boolean | no | - | Only used when file_format is xml, specifies Whether to process data using the tag attribute format. [...]
+| parquet_avro_write_timestamp_as_int96 | boolean | no | false | Only used when file_format is parquet. [...]
+| parquet_avro_write_fixed_as_int96 | array | no | - | Only used when file_format is parquet. [...]
+| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. [...]
### Tips
diff --git a/docs/en/connector-v2/sink/LocalFile.md b/docs/en/connector-v2/sink/LocalFile.md
index d2ab6ea8e1..b0d6b45bd1 100644
--- a/docs/en/connector-v2/sink/LocalFile.md
+++ b/docs/en/connector-v2/sink/LocalFile.md
@@ -32,32 +32,34 @@ By default, we use 2PC commit to ensure `exactly-once`
## Options
-| Name | Type | Required | Default | Description |
-|----------------------------------|---------|----------|--------------------------------------------|---------------------------------------------------------------------------------------------------|
-| path | string | yes | - | |
-| tmp_path | string | no | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. |
-| custom_filename | boolean | no | false | Whether you need custom the filename |
-| file_name_expression | string | no | "${transactionId}" | Only used when custom_filename is true |
-| filename_time_format | string | no | "yyyy.MM.dd" | Only used when custom_filename is true |
-| file_format_type | string | no | "csv" | |
-| field_delimiter | string | no | '\001' | Only used when file_format_type is text |
-| row_delimiter | string | no | "\n" | Only used when file_format_type is text |
-| have_partition | boolean | no | false | Whether you need processing partitions. |
-| partition_by | array | no | - | Only used then have_partition is true |
-| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | Only used then have_partition is true |
-| is_partition_field_write_in_file | boolean | no | false | Only used then have_partition is true |
-| sink_columns | array | no | | When this parameter is empty, all fields are sink columns |
-| is_enable_transaction | boolean | no | true | |
-| batch_size | int | no | 1000000 | |
-| compress_codec | string | no | none | |
-| common-options | object | no | - | |
-| max_rows_in_memory | int | no | - | Only used when file_format_type is excel. |
-| sheet_name | string | no | Sheet${Random number} | Only used when file_format_type is excel. |
-| xml_root_tag | string | no | RECORDS | Only used when file_format is xml. |
-| xml_row_tag | string | no | RECORD | Only used when file_format is xml. |
-| xml_use_attr_format | boolean | no | - | Only used when file_format is xml. |
-| enable_header_write | boolean | no | false | Only used when file_format_type is text,csv.<br/> false:don't write header,true:write header. |
-| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
+| Name | Type | Required | Default | Description |
+|---------------------------------------|---------|----------|--------------------------------------------|---------------------------------------------------------------------------------------------------|
+| path | string | yes | - | |
+| tmp_path | string | no | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. |
+| custom_filename | boolean | no | false | Whether you need custom the filename |
+| file_name_expression | string | no | "${transactionId}" | Only used when custom_filename is true |
+| filename_time_format | string | no | "yyyy.MM.dd" | Only used when custom_filename is true |
+| file_format_type | string | no | "csv" | |
+| field_delimiter | string | no | '\001' | Only used when file_format_type is text |
+| row_delimiter | string | no | "\n" | Only used when file_format_type is text |
+| have_partition | boolean | no | false | Whether you need processing partitions. |
+| partition_by | array | no | - | Only used then have_partition is true |
+| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | Only used then have_partition is true |
+| is_partition_field_write_in_file | boolean | no | false | Only used then have_partition is true |
+| sink_columns | array | no | | When this parameter is empty, all fields are sink columns |
+| is_enable_transaction | boolean | no | true | |
+| batch_size | int | no | 1000000 | |
+| compress_codec | string | no | none | |
+| common-options | object | no | - | |
+| max_rows_in_memory | int | no | - | Only used when file_format_type is excel. |
+| sheet_name | string | no | Sheet${Random number} | Only used when file_format_type is excel. |
+| xml_root_tag | string | no | RECORDS | Only used when file_format is xml. |
+| xml_row_tag | string | no | RECORD | Only used when file_format is xml. |
+| xml_use_attr_format | boolean | no | - | Only used when file_format is xml. |
+| parquet_avro_write_timestamp_as_int96 | boolean | no | false | Only used when file_format is parquet. |
+| parquet_avro_write_fixed_as_int96 | array | no | - | Only used when file_format is parquet. |
+| enable_header_write | boolean | no | false | Only used when file_format_type is text,csv.<br/> false:don't write header,true:write header. |
+| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
### path [string]
@@ -186,6 +188,14 @@ Specifies the tag name of the data rows within the XML file.
Specifies Whether to process data using the tag attribute format.
+### parquet_avro_write_timestamp_as_int96 [boolean]
+
+Support writing Parquet INT96 from a timestamp, only valid for parquet files.
+
+### parquet_avro_write_fixed_as_int96 [array]
+
+Support writing Parquet INT96 from a 12-byte field, only valid for parquet files.
+
### enable_header_write [boolean]
Only used when file_format_type is text,csv.false:don't write header,true:write header.
diff --git a/docs/en/connector-v2/sink/OssFile.md b/docs/en/connector-v2/sink/OssFile.md
index 41dbead736..aef2bb11c0 100644
--- a/docs/en/connector-v2/sink/OssFile.md
+++ b/docs/en/connector-v2/sink/OssFile.md
@@ -85,35 +85,37 @@ If write to `csv`, `text` file type, All column will be string.
## Options
-| Name | Type | Required | Default | Description |
-|----------------------------------|---------|----------|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------|
-| path | string | yes | The oss path to write file in. | |
-| tmp_path | string | no | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. Need a OSS dir. |
-| bucket | string | yes | - | |
-| access_key | string | yes | - | |
-| access_secret | string | yes | - | |
-| endpoint | string | yes | - | |
-| custom_filename | boolean | no | false | Whether you need custom the filename |
-| file_name_expression | string | no | "${transactionId}" | Only used when custom_filename is true |
-| filename_time_format | string | no | "yyyy.MM.dd" | Only used when custom_filename is true |
-| file_format_type | string | no | "csv" | |
-| field_delimiter | string | no | '\001' | Only used when file_format_type is text |
-| row_delimiter | string | no | "\n" | Only used when file_format_type is text |
-| have_partition | boolean | no | false | Whether you need processing partitions. |
-| partition_by | array | no | - | Only used then have_partition is true |
-| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | Only used then have_partition is true |
-| is_partition_field_write_in_file | boolean | no | false | Only used then have_partition is true |
-| sink_columns | array | no | | When this parameter is empty, all fields are sink columns |
-| is_enable_transaction | boolean | no | true | |
-| batch_size | int | no | 1000000 | |
-| compress_codec | string | no | none | |
-| common-options | object | no | - | |
-| max_rows_in_memory | int | no | - | Only used when file_format_type is excel. |
-| sheet_name | string | no | Sheet${Random number} | Only used when file_format_type is excel. |
-| xml_root_tag | string | no | RECORDS | Only used when file_format is xml. |
-| xml_row_tag | string | no | RECORD | Only used when file_format is xml. |
-| xml_use_attr_format | boolean | no | - | Only used when file_format is xml. |
-| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
+| Name | Type | Required | Default | Description |
+|---------------------------------------|---------|----------|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------|
+| path | string | yes | The oss path to write file in. | |
+| tmp_path | string | no | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. Need a OSS dir. |
+| bucket | string | yes | - | |
+| access_key | string | yes | - | |
+| access_secret | string | yes | - | |
+| endpoint | string | yes | - | |
+| custom_filename | boolean | no | false | Whether you need custom the filename |
+| file_name_expression | string | no | "${transactionId}" | Only used when custom_filename is true |
+| filename_time_format | string | no | "yyyy.MM.dd" | Only used when custom_filename is true |
+| file_format_type | string | no | "csv" | |
+| field_delimiter | string | no | '\001' | Only used when file_format_type is text |
+| row_delimiter | string | no | "\n" | Only used when file_format_type is text |
+| have_partition | boolean | no | false | Whether you need processing partitions. |
+| partition_by | array | no | - | Only used then have_partition is true |
+| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | Only used then have_partition is true |
+| is_partition_field_write_in_file | boolean | no | false | Only used then have_partition is true |
+| sink_columns | array | no | | When this parameter is empty, all fields are sink columns |
+| is_enable_transaction | boolean | no | true | |
+| batch_size | int | no | 1000000 | |
+| compress_codec | string | no | none | |
+| common-options | object | no | - | |
+| max_rows_in_memory | int | no | - | Only used when file_format_type is excel. |
+| sheet_name | string | no | Sheet${Random number} | Only used when file_format_type is excel. |
+| xml_root_tag | string | no | RECORDS | Only used when file_format is xml. |
+| xml_row_tag | string | no | RECORD | Only used when file_format is xml. |
+| xml_use_attr_format | boolean | no | - | Only used when file_format is xml. |
+| parquet_avro_write_timestamp_as_int96 | boolean | no | false | Only used when file_format is parquet. |
+| parquet_avro_write_fixed_as_int96 | array | no | - | Only used when file_format is parquet. |
+| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
### path [string]
@@ -258,6 +260,14 @@ Specifies the tag name of the data rows within the XML file.
Specifies Whether to process data using the tag attribute format.
+### parquet_avro_write_timestamp_as_int96 [boolean]
+
+Support writing Parquet INT96 from a timestamp, only valid for parquet files.
+
+### parquet_avro_write_fixed_as_int96 [array]
+
+Support writing Parquet INT96 from a 12-byte field, only valid for parquet files.
+
### encoding [string]
Only used when file_format_type is json,text,csv,xml.
diff --git a/docs/en/connector-v2/sink/OssJindoFile.md b/docs/en/connector-v2/sink/OssJindoFile.md
index cab0455613..80e6cf775c 100644
--- a/docs/en/connector-v2/sink/OssJindoFile.md
+++ b/docs/en/connector-v2/sink/OssJindoFile.md
@@ -38,35 +38,37 @@ By default, we use 2PC commit to ensure `exactly-once`
## Options
-| Name | Type | Required | Default | Description |
-|----------------------------------|---------|----------|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------|
-| path | string | yes | - | |
-| tmp_path | string | no | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. Need a OSS dir. |
-| bucket | string | yes | - | |
-| access_key | string | yes | - | |
-| access_secret | string | yes | - | |
-| endpoint | string | yes | - | |
-| custom_filename | boolean | no | false | Whether you need custom the filename |
-| file_name_expression | string | no | "${transactionId}" | Only used when custom_filename is true |
-| filename_time_format | string | no | "yyyy.MM.dd" | Only used when custom_filename is true |
-| file_format_type | string | no | "csv" | |
-| field_delimiter | string | no | '\001' | Only used when file_format_type is text |
-| row_delimiter | string | no | "\n" | Only used when file_format_type is text |
-| have_partition | boolean | no | false | Whether you need processing partitions. |
-| partition_by | array | no | - | Only used then have_partition is true |
-| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | Only used then have_partition is true |
-| is_partition_field_write_in_file | boolean | no | false | Only used then have_partition is true |
-| sink_columns | array | no | | When this parameter is empty, all fields are sink columns |
-| is_enable_transaction | boolean | no | true | |
-| batch_size | int | no | 1000000 | |
-| compress_codec | string | no | none | |
-| common-options | object | no | - | |
-| max_rows_in_memory | int | no | - | Only used when file_format_type is excel. |
-| sheet_name | string | no | Sheet${Random number} | Only used when file_format_type is excel. |
-| xml_root_tag | string | no | RECORDS | Only used when file_format is xml. |
-| xml_row_tag | string | no | RECORD | Only used when file_format is xml. |
-| xml_use_attr_format | boolean | no | - | Only used when file_format is xml. |
-| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
+| Name | Type | Required | Default | Description |
+|---------------------------------------|---------|----------|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------|
+| path | string | yes | - | |
+| tmp_path | string | no | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. Need a OSS dir. |
+| bucket | string | yes | - | |
+| access_key | string | yes | - | |
+| access_secret | string | yes | - | |
+| endpoint | string | yes | - | |
+| custom_filename | boolean | no | false | Whether you need custom the filename |
+| file_name_expression | string | no | "${transactionId}" | Only used when custom_filename is true |
+| filename_time_format | string | no | "yyyy.MM.dd" | Only used when custom_filename is true |
+| file_format_type | string | no | "csv" | |
+| field_delimiter | string | no | '\001' | Only used when file_format_type is text |
+| row_delimiter | string | no | "\n" | Only used when file_format_type is text |
+| have_partition | boolean | no | false | Whether you need processing partitions. |
+| partition_by | array | no | - | Only used then have_partition is true |
+| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | Only used then have_partition is true |
+| is_partition_field_write_in_file | boolean | no | false | Only used then have_partition is true |
+| sink_columns | array | no | | When this parameter is empty, all fields are sink columns |
+| is_enable_transaction | boolean | no | true | |
+| batch_size | int | no | 1000000 | |
+| compress_codec | string | no | none | |
+| common-options | object | no | - | |
+| max_rows_in_memory | int | no | - | Only used when file_format_type is excel. |
+| sheet_name | string | no | Sheet${Random number} | Only used when file_format_type is excel. |
+| xml_root_tag | string | no | RECORDS | Only used when file_format is xml. |
+| xml_row_tag | string | no | RECORD | Only used when file_format is xml. |
+| xml_use_attr_format | boolean | no | - | Only used when file_format is xml. |
+| parquet_avro_write_timestamp_as_int96 | boolean | no | false | Only used when file_format is parquet. |
+| parquet_avro_write_fixed_as_int96 | array | no | - | Only used when file_format is parquet. |
+| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
### path [string]
@@ -211,6 +213,14 @@ Specifies the tag name of the data rows within the XML file.
Specifies Whether to process data using the tag attribute format.
+### parquet_avro_write_timestamp_as_int96 [boolean]
+
+Support writing Parquet INT96 from a timestamp, only valid for parquet files.
+
+### parquet_avro_write_fixed_as_int96 [array]
+
+Support writing Parquet INT96 from a 12-byte field, only valid for parquet files.
+
### encoding [string]
Only used when file_format_type is json,text,csv,xml.
diff --git a/docs/en/connector-v2/sink/S3File.md b/docs/en/connector-v2/sink/S3File.md
index b59bc078e3..c25975a860 100644
--- a/docs/en/connector-v2/sink/S3File.md
+++ b/docs/en/connector-v2/sink/S3File.md
@@ -92,39 +92,41 @@ If write to `csv`, `text` file type, All column will be string.
## Sink Options
-| name | type | required | default value | Description |
-|----------------------------------|---------|----------|-------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| path | string | yes | - | |
-| tmp_path | string | no | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. Need a S3 dir. |
-| bucket | string | yes | - | |
-| fs.s3a.endpoint | string | yes | - | |
-| fs.s3a.aws.credentials.provider | string | yes | com.amazonaws.auth.InstanceProfileCredentialsProvider | The way to authenticate s3a. We only support `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider` and `com.amazonaws.auth.InstanceProfileCredentialsProvider` now. |
-| access_key | string | no | - | Only used when fs.s3a.aws.credentials.provider = org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider |
-| access_secret | string | no | - | Only used when fs.s3a.aws.credentials.provider = org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider |
-| custom_filename | boolean | no | false | Whether you need custom the filename |
-| file_name_expression | string | no | "${transactionId}" | Only used when custom_filename is true |
-| filename_time_format | string | no | "yyyy.MM.dd" | Only used when custom_filename is true |
-| file_format_type | string | no | "csv" | |
-| field_delimiter | string | no | '\001' | Only used when file_format is text |
-| row_delimiter | string | no | "\n" | Only used when file_format is text |
-| have_partition | boolean | no | false | Whether you need processing partitions. |
-| partition_by | array | no | - | Only used when have_partition is true |
-| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | Only used when have_partition is true |
-| is_partition_field_write_in_file | boolean | no | false | Only used when have_partition is true |
-| sink_columns | array | no | | When this parameter is empty, all fields are sink columns |
-| is_enable_transaction | boolean | no | true | |
-| batch_size | int | no | 1000000 | |
-| compress_codec | string | no | none | |
-| common-options | object | no | - | |
-| max_rows_in_memory | int | no | - | Only used when file_format is excel. |
-| sheet_name | string | no | Sheet${Random number} | Only used when file_format is excel. |
-| xml_root_tag | string | no | RECORDS | Only used when file_format is xml, specifies the tag name of the root element within the XML file. |
-| xml_row_tag | string | no | RECORD | Only used when file_format is xml, specifies the tag name of the data rows within the XML file |
-| xml_use_attr_format | boolean | no | - | Only used when file_format is xml, specifies Whether to process data using the tag attribute format. |
-| hadoop_s3_properties | map | no | | If you need to add a other option, you could add it here and refer to this [link](https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/index.html) |
-| schema_save_mode | Enum | no | CREATE_SCHEMA_WHEN_NOT_EXIST | Before turning on the synchronous task, do different treatment of the target path |
-| data_save_mode | Enum | no | APPEND_DATA | Before opening the synchronous task, the data file in the target path is differently processed |
-| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
+| name | type | required | default value | Description |
+|---------------------------------------|---------|----------|-------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| path | string | yes | - | |
+| tmp_path | string | no | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. Need a S3 dir. |
+| bucket | string | yes | - | |
+| fs.s3a.endpoint | string | yes | - | |
+| fs.s3a.aws.credentials.provider | string | yes | com.amazonaws.auth.InstanceProfileCredentialsProvider | The way to authenticate s3a. We only support `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider` and `com.amazonaws.auth.InstanceProfileCredentialsProvider` now. |
+| access_key | string | no | - | Only used when fs.s3a.aws.credentials.provider = org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider |
+| access_secret | string | no | - | Only used when fs.s3a.aws.credentials.provider = org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider |
+| custom_filename | boolean | no | false | Whether you need custom the filename |
+| file_name_expression | string | no | "${transactionId}" | Only used when custom_filename is true |
+| filename_time_format | string | no | "yyyy.MM.dd" | Only used when custom_filename is true |
+| file_format_type | string | no | "csv" | |
+| field_delimiter | string | no | '\001' | Only used when file_format is text |
+| row_delimiter | string | no | "\n" | Only used when file_format is text |
+| have_partition | boolean | no | false | Whether you need processing partitions. |
+| partition_by | array | no | - | Only used when have_partition is true |
+| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | Only used when have_partition is true |
+| is_partition_field_write_in_file | boolean | no | false | Only used when have_partition is true |
+| sink_columns | array | no | | When this parameter is empty, all fields are sink columns |
+| is_enable_transaction | boolean | no | true | |
+| batch_size | int | no | 1000000 | |
+| compress_codec | string | no | none | |
+| common-options | object | no | - | |
+| max_rows_in_memory | int | no | - | Only used when file_format is excel. |
+| sheet_name | string | no | Sheet${Random number} | Only used when file_format is excel. |
+| xml_root_tag | string | no | RECORDS | Only used when file_format is xml, specifies the tag name of the root element within the XML file. |
+| xml_row_tag | string | no | RECORD | Only used when file_format is xml, specifies the tag name of the data rows within the XML file |
+| xml_use_attr_format | boolean | no | - | Only used when file_format is xml, specifies Whether to process data using the tag attribute format. |
+| parquet_avro_write_timestamp_as_int96 | boolean | no | false | Only used when file_format is parquet. |
+| parquet_avro_write_fixed_as_int96 | array | no | - | Only used when file_format is parquet. |
+| hadoop_s3_properties | map | no | | If you need to add a other option, you could add it here and refer to this [link](https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/index.html) |
+| schema_save_mode | Enum | no | CREATE_SCHEMA_WHEN_NOT_EXIST | Before turning on the synchronous task, do different treatment of the target path |
+| data_save_mode | Enum | no | APPEND_DATA | Before opening the synchronous task, the data file in the target path is differently processed |
+| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
### path [string]
@@ -264,6 +266,14 @@ Specifies the tag name of the data rows within the XML file.
Specifies Whether to process data using the tag attribute format.
+### parquet_avro_write_timestamp_as_int96 [boolean]
+
+Support writing Parquet INT96 from a timestamp, only valid for parquet files.
+
+### parquet_avro_write_fixed_as_int96 [array]
+
+Support writing Parquet INT96 from a 12-byte field, only valid for parquet files.
+
### schema_save_mode[Enum]
Before turning on the synchronous task, do different treatment of the target path.
diff --git a/docs/en/connector-v2/sink/SftpFile.md b/docs/en/connector-v2/sink/SftpFile.md
index bba650f6a1..d06292bafd 100644
--- a/docs/en/connector-v2/sink/SftpFile.md
+++ b/docs/en/connector-v2/sink/SftpFile.md
@@ -32,35 +32,37 @@ By default, we use 2PC commit to ensure `exactly-once`
## Options
-| name | type | required | default value | remarks |
-|----------------------------------|---------|----------|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------|
-| host | string | yes | - | |
-| port | int | yes | - | |
-| username | string | yes | - | |
-| password | string | yes | - | |
-| path | string | yes | - | |
-| tmp_path | string | yes | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. Need a FTP dir. |
-| custom_filename | boolean | no | false | Whether you need custom the filename |
-| file_name_expression | string | no | "${transactionId}" | Only used when custom_filename is true |
-| filename_time_format | string | no | "yyyy.MM.dd" | Only used when custom_filename is true |
-| file_format_type | string | no | "csv" | |
-| field_delimiter | string | no | '\001' | Only used when file_format_type is text |
-| row_delimiter | string | no | "\n" | Only used when file_format_type is text |
-| have_partition | boolean | no | false | Whether you need processing partitions. |
-| partition_by | array | no | - | Only used then have_partition is true |
-| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | Only used then have_partition is true |
-| is_partition_field_write_in_file | boolean | no | false | Only used then have_partition is true |
-| sink_columns | array | no | | When this parameter is empty, all fields are sink columns |
-| is_enable_transaction | boolean | no | true | |
-| batch_size | int | no | 1000000 | |
-| compress_codec | string | no | none | |
-| common-options | object | no | - | |
-| max_rows_in_memory | int | no | - | Only used when file_format_type is excel. |
-| sheet_name | string | no | Sheet${Random number} | Only used when file_format_type is excel. |
-| xml_root_tag | string | no | RECORDS | Only used when file_format is xml. |
-| xml_row_tag | string | no | RECORD | Only used when file_format is xml. |
-| xml_use_attr_format | boolean | no | - | Only used when file_format is xml. |
-| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
+| name | type | required | default value | remarks |
+|---------------------------------------|---------|----------|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------|
+| host | string | yes | - | |
+| port | int | yes | - | |
+| username | string | yes | - | |
+| password | string | yes | - | |
+| path | string | yes | - | |
+| tmp_path | string | yes | /tmp/seatunnel | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. Need a FTP dir. |
+| custom_filename | boolean | no | false | Whether you need custom the filename |
+| file_name_expression | string | no | "${transactionId}" | Only used when custom_filename is true |
+| filename_time_format | string | no | "yyyy.MM.dd" | Only used when custom_filename is true |
+| file_format_type | string | no | "csv" | |
+| field_delimiter | string | no | '\001' | Only used when file_format_type is text |
+| row_delimiter | string | no | "\n" | Only used when file_format_type is text |
+| have_partition | boolean | no | false | Whether you need processing partitions. |
+| partition_by | array | no | - | Only used then have_partition is true |
+| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | Only used then have_partition is true |
+| is_partition_field_write_in_file | boolean | no | false | Only used then have_partition is true |
+| sink_columns | array | no | | When this parameter is empty, all fields are sink columns |
+| is_enable_transaction | boolean | no | true | |
+| batch_size | int | no | 1000000 | |
+| compress_codec | string | no | none | |
+| common-options | object | no | - | |
+| max_rows_in_memory | int | no | - | Only used when file_format_type is excel. |
+| sheet_name | string | no | Sheet${Random number} | Only used when file_format_type is excel. |
+| xml_root_tag | string | no | RECORDS | Only used when file_format is xml. |
+| xml_row_tag | string | no | RECORD | Only used when file_format is xml. |
+| xml_use_attr_format | boolean | no | - | Only used when file_format is xml. |
+| parquet_avro_write_timestamp_as_int96 | boolean | no | false | Only used when file_format is parquet. |
+| parquet_avro_write_fixed_as_int96 | array | no | - | Only used when file_format is parquet. |
+| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
### host [string]
@@ -205,6 +207,14 @@ Specifies the tag name of the data rows within the XML file.
Specifies Whether to process data using the tag attribute format.
+### parquet_avro_write_timestamp_as_int96 [boolean]
+
+Support writing Parquet INT96 from a timestamp, only valid for parquet files.
+
+### parquet_avro_write_fixed_as_int96 [array]
+
+Support writing Parquet INT96 from a 12-byte field, only valid for parquet files.
+
### encoding [string]
Only used when file_format_type is json,text,csv,xml.
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSinkConfig.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSinkConfig.java
index 394ccc5ceb..48a2d2b436 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSinkConfig.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSinkConfig.java
@@ -25,6 +25,7 @@ import org.apache.seatunnel.common.utils.TimeUtils;
import org.apache.seatunnel.format.text.constant.TextFormatConstant;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
public class BaseSinkConfig {
@@ -278,4 +279,18 @@ public class BaseSinkConfig {
.booleanType()
.defaultValue(false)
.withDescription("false:dont write header,true:write header");
+
+ public static final Option<Boolean> PARQUET_AVRO_WRITE_TIMESTAMP_AS_INT96 =
+ Options.key("parquet_avro_write_timestamp_as_int96")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "Support writing Parquet INT96 from a timestamp, only valid for parquet files.");
+
+ public static final Option<List<String>> PARQUET_AVRO_WRITE_FIXED_AS_INT96 =
+ Options.key("parquet_avro_write_fixed_as_int96")
+ .listType(String.class)
+ .defaultValue(Collections.emptyList())
+ .withDescription(
+ "Support writing Parquet INT96 from a 12-byte field, only valid for parquet files.");
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/HadoopConf.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/HadoopConf.java
index 4e6499f4de..3a88fa3b3d 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/HadoopConf.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/HadoopConf.java
@@ -29,7 +29,6 @@ import java.util.Map;
import static org.apache.parquet.avro.AvroReadSupport.READ_INT96_AS_FIXED;
import static org.apache.parquet.avro.AvroSchemaConverter.ADD_LIST_ELEMENT_RECORDS;
-import static org.apache.parquet.avro.AvroWriteSupport.WRITE_FIXED_AS_INT96;
import static org.apache.parquet.avro.AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE;
@Data
@@ -86,7 +85,6 @@ public class HadoopConf implements Serializable {
public Configuration toConfiguration() {
Configuration configuration = new Configuration();
configuration.setBoolean(READ_INT96_AS_FIXED, true);
- configuration.setBoolean(WRITE_FIXED_AS_INT96, true);
configuration.setBoolean(ADD_LIST_ELEMENT_RECORDS, false);
configuration.setBoolean(WRITE_OLD_LIST_STRUCTURE, true);
configuration.setBoolean(getHdfsImplDisableCacheKey(), true);
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/config/FileSinkConfig.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/config/FileSinkConfig.java
index 883a45d9cb..577b62defb 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/config/FileSinkConfig.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/config/FileSinkConfig.java
@@ -80,6 +80,12 @@ public class FileSinkConfig extends BaseFileSinkConfig implements PartitionConfi
private Boolean xmlUseAttrFormat;
+ private Boolean parquetWriteTimestampAsInt96 =
+ BaseSinkConfig.PARQUET_AVRO_WRITE_TIMESTAMP_AS_INT96.defaultValue();
+
+ private List<String> parquetAvroWriteFixedAsInt96 =
+ BaseSinkConfig.PARQUET_AVRO_WRITE_FIXED_AS_INT96.defaultValue();
+
public FileSinkConfig(@NonNull Config config, @NonNull SeaTunnelRowType seaTunnelRowTypeInfo) {
super(config);
checkArgument(
@@ -217,5 +223,20 @@ public class FileSinkConfig extends BaseFileSinkConfig implements PartitionConfi
this.xmlRowTag = config.getString(BaseSinkConfig.XML_ROW_TAG.key());
}
}
+
+ if (FileFormat.PARQUET
+ .name()
+ .equalsIgnoreCase(config.getString(BaseSinkConfig.FILE_FORMAT_TYPE.key()))) {
+ if (config.hasPath(BaseSinkConfig.PARQUET_AVRO_WRITE_TIMESTAMP_AS_INT96.key())) {
+ this.parquetWriteTimestampAsInt96 =
+ config.getBoolean(
+ BaseSinkConfig.PARQUET_AVRO_WRITE_TIMESTAMP_AS_INT96.key());
+ }
+ if (config.hasPath(BaseSinkConfig.PARQUET_AVRO_WRITE_FIXED_AS_INT96.key())) {
+ this.parquetAvroWriteFixedAsInt96 =
+ config.getStringList(
+ BaseSinkConfig.PARQUET_AVRO_WRITE_FIXED_AS_INT96.key());
+ }
+ }
}
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java
index fd1c837571..b0f873296c 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java
@@ -23,6 +23,7 @@ import org.apache.seatunnel.api.table.type.MapType;
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.api.table.type.SqlType;
import org.apache.seatunnel.common.exception.CommonError;
import org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated;
import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf;
@@ -35,10 +36,13 @@ import org.apache.avro.data.TimeConversions;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.GenericRecordBuilder;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.avro.AvroParquetWriter;
import org.apache.parquet.avro.AvroSchemaConverter;
+import org.apache.parquet.avro.AvroWriteSupport;
import org.apache.parquet.column.ParquetProperties;
+import org.apache.parquet.example.data.simple.NanoTime;
import org.apache.parquet.hadoop.ParquetFileWriter;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.util.HadoopOutputFile;
@@ -54,11 +58,19 @@ import lombok.NonNull;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
+import java.time.temporal.JulianFields;
import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -66,6 +78,7 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy {
private final LinkedHashMap<String, ParquetWriter<GenericRecord>> beingWrittenWriter;
private AvroSchemaConverter schemaConverter;
private Schema schema;
+ private Set<String> writePathsAsInt96;
public static final int[] PRECISION_TO_BYTE_COUNT = new int[38];
static {
@@ -84,7 +97,22 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy {
@Override
public void init(HadoopConf conf, String jobId, String uuidPrefix, int subTaskIndex) {
super.init(conf, jobId, uuidPrefix, subTaskIndex);
- schemaConverter = new AvroSchemaConverter(getConfiguration(hadoopConf));
+ Configuration configuration = getConfiguration(hadoopConf);
+ writePathsAsInt96 = new HashSet<>(fileSinkConfig.getParquetAvroWriteFixedAsInt96());
+ if (fileSinkConfig.getParquetWriteTimestampAsInt96()) {
+ List<String> timestampFields = new ArrayList<>();
+ for (int i = 0; i < seaTunnelRowType.getTotalFields(); i++) {
+ if (SqlType.TIMESTAMP.equals(seaTunnelRowType.getFieldType(i).getSqlType())) {
+ timestampFields.add(seaTunnelRowType.getFieldName(i));
+ }
+ }
+ writePathsAsInt96.addAll(timestampFields);
+ }
+ if (!writePathsAsInt96.isEmpty()) {
+ configuration.set(
+ AvroWriteSupport.WRITE_FIXED_AS_INT96, String.join(",", writePathsAsInt96));
+ }
+ schemaConverter = new AvroSchemaConverter(configuration);
}
@Override
@@ -98,7 +126,7 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy {
Object field = seaTunnelRow.getField(integer);
recordBuilder.set(
fieldName.toLowerCase(),
- resolveObject(field, seaTunnelRowType.getFieldType(integer)));
+ resolveObject(fieldName, field, seaTunnelRowType.getFieldType(integer)));
}
GenericData.Record record = recordBuilder.build();
try {
@@ -142,12 +170,18 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy {
return hadoopFileSystemProxy.doWithHadoopAuth(
(configuration, userGroupInformation) -> {
try {
+ if (!writePathsAsInt96.isEmpty()) {
+ configuration.set(
+ AvroWriteSupport.WRITE_FIXED_AS_INT96,
+ String.join(",", writePathsAsInt96));
+ }
HadoopOutputFile outputFile =
HadoopOutputFile.fromPath(path, getConfiguration(hadoopConf));
ParquetWriter<GenericRecord> newWriter =
AvroParquetWriter.<GenericRecord>builder(outputFile)
.withWriteMode(ParquetFileWriter.Mode.OVERWRITE)
.withDataModel(dataModel)
+ .withConf(configuration)
// use parquet v1 to improve compatibility
.withWriterVersion(
ParquetProperties.WriterVersion.PARQUET_1_0)
@@ -169,7 +203,7 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy {
return writer;
}
- private Object resolveObject(Object data, SeaTunnelDataType<?> seaTunnelDataType) {
+ private Object resolveObject(String name, Object data, SeaTunnelDataType<?> seaTunnelDataType) {
if (data == null) {
return null;
}
@@ -179,7 +213,7 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy {
((ArrayType<?, ?>) seaTunnelDataType).getElementType();
ArrayList<Object> records = new ArrayList<>(((Object[]) data).length);
for (Object object : (Object[]) data) {
- Object resolvedObject = resolveObject(object, elementType);
+ Object resolvedObject = resolveObject(name, object, elementType);
records.add(resolvedObject);
}
return records;
@@ -197,11 +231,36 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy {
case DATE:
return data;
case TIMESTAMP:
+ if (writePathsAsInt96.contains(name)) {
+ LocalDateTime localDateTime = (LocalDateTime) data;
+ Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+ calendar.setTime(
+ Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant()));
+ int julianDays =
+ (int)
+ JulianFields.JULIAN_DAY.getFrom(
+ LocalDate.of(
+ calendar.get(Calendar.YEAR),
+ calendar.get(Calendar.MONTH) + 1,
+ calendar.get(Calendar.DAY_OF_MONTH)));
+ long timeOfDayNanos =
+ TimeUnit.HOURS.toNanos(calendar.get(Calendar.HOUR_OF_DAY))
+ + TimeUnit.MINUTES.toNanos(calendar.get(Calendar.MINUTE))
+ + TimeUnit.SECONDS.toNanos(calendar.get(Calendar.SECOND))
+ + TimeUnit.MILLISECONDS.toNanos(
+ calendar.get(Calendar.MILLISECOND));
+ NanoTime nanoTime = new NanoTime(julianDays, timeOfDayNanos);
+ return new GenericData.Fixed(
+ schema.getField(name).schema(), nanoTime.toBinary().getBytes());
+ }
return ((LocalDateTime) data)
.atZone(ZoneId.systemDefault())
.toInstant()
.toEpochMilli();
case BYTES:
+ if (writePathsAsInt96.contains(name)) {
+ return new GenericData.Fixed(schema.getField(name).schema(), (byte[]) data);
+ }
return ByteBuffer.wrap((byte[]) data);
case ROW:
SeaTunnelRow seaTunnelRow = (SeaTunnelRow) data;
@@ -219,7 +278,7 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy {
for (int i = 0; i < fieldNames.length; i++) {
recordBuilder.set(
fieldNames[i].toLowerCase(),
- resolveObject(seaTunnelRow.getField(i), fieldTypes[i]));
+ resolveObject(fieldNames[i], seaTunnelRow.getField(i), fieldTypes[i]));
}
return recordBuilder.build();
default:
@@ -232,7 +291,7 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy {
}
}
- public static Type seaTunnelDataType2ParquetDataType(
+ public Type seaTunnelDataType2ParquetDataType(
String fieldName, SeaTunnelDataType<?> seaTunnelDataType) {
switch (seaTunnelDataType.getSqlType()) {
case ARRAY:
@@ -291,6 +350,11 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy {
PrimitiveType.PrimitiveTypeName.INT64, Type.Repetition.OPTIONAL)
.named(fieldName);
case TIMESTAMP:
+ if (writePathsAsInt96.contains(fieldName)) {
+ return Types.primitive(
+ PrimitiveType.PrimitiveTypeName.INT96, Type.Repetition.OPTIONAL)
+ .named(fieldName);
+ }
return Types.primitive(
PrimitiveType.PrimitiveTypeName.INT64, Type.Repetition.OPTIONAL)
.as(OriginalType.TIMESTAMP_MILLIS)
@@ -313,6 +377,13 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy {
.scale(scale)
.named(fieldName);
case BYTES:
+ if (writePathsAsInt96.contains(fieldName)) {
+ return Types.primitive(
+ PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
+ Type.Repetition.OPTIONAL)
+ .length(12)
+ .named(fieldName);
+ }
return Types.primitive(
PrimitiveType.PrimitiveTypeName.BINARY, Type.Repetition.OPTIONAL)
.named(fieldName);
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java
new file mode 100644
index 0000000000..236d6f5a03
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.file.writer;
+
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
+
+import org.apache.seatunnel.api.source.Collector;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.LocalTimeType;
+import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.connectors.seatunnel.file.config.FileFormat;
+import org.apache.seatunnel.connectors.seatunnel.file.sink.config.FileSinkConfig;
+import org.apache.seatunnel.connectors.seatunnel.file.sink.writer.ParquetWriteStrategy;
+import org.apache.seatunnel.connectors.seatunnel.file.source.reader.ParquetReadStrategy;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.metadata.FileMetaData;
+import org.apache.parquet.hadoop.util.HadoopInputFile;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledOnOs;
+import org.junit.jupiter.api.condition.OS;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
+
+@Slf4j
+public class ParquetWriteStrategyTest {
+ private static final String TMP_PATH = "file:///tmp/seatunnel/parquet/int96/test";
+
+ @DisabledOnOs(OS.WINDOWS)
+ @Test
+ public void testParquetWriteInt96() throws Exception {
+ Map<String, Object> writeConfig = new HashMap<>();
+ writeConfig.put("tmp_path", TMP_PATH);
+ writeConfig.put("path", "file:///tmp/seatunnel/parquet/int96");
+ writeConfig.put("file_format_type", FileFormat.PARQUET.name());
+ writeConfig.put("parquet_avro_write_timestamp_as_int96", "true");
+ writeConfig.put("parquet_avro_write_fixed_as_int96", Arrays.asList("f3_bytes"));
+
+ SeaTunnelRowType writeRowType =
+ new SeaTunnelRowType(
+ new String[] {"f1_text", "f2_timestamp", "f3_bytes"},
+ new SeaTunnelDataType[] {
+ BasicType.STRING_TYPE,
+ LocalTimeType.LOCAL_DATE_TIME_TYPE,
+ PrimitiveByteArrayType.INSTANCE
+ });
+ FileSinkConfig writeSinkConfig =
+ new FileSinkConfig(ConfigFactory.parseMap(writeConfig), writeRowType);
+ ParquetWriteStrategy writeStrategy = new ParquetWriteStrategy(writeSinkConfig);
+ ParquetReadStrategyTest.LocalConf hadoopConf =
+ new ParquetReadStrategyTest.LocalConf(FS_DEFAULT_NAME_DEFAULT);
+ writeStrategy.setSeaTunnelRowTypeInfo(writeRowType);
+ writeStrategy.init(hadoopConf, "test1", "test1", 0);
+ writeStrategy.beginTransaction(1L);
+ writeStrategy.write(
+ new SeaTunnelRow(new Object[] {"test", LocalDateTime.now(), new byte[12]}));
+ writeStrategy.finishAndCloseFile();
+ writeStrategy.close();
+
+ ParquetReadStrategy readStrategy = new ParquetReadStrategy();
+ readStrategy.init(hadoopConf);
+ List<String> readFiles = readStrategy.getFileNamesByPath(TMP_PATH);
+ Assertions.assertEquals(1, readFiles.size());
+ String readFilePath = readFiles.get(0);
+ try (ParquetFileReader reader =
+ ParquetFileReader.open(
+ HadoopInputFile.fromPath(
+ new org.apache.hadoop.fs.Path(readFilePath),
+ new Configuration()))) {
+ FileMetaData metadata = reader.getFooter().getFileMetaData();
+ Type f1Type = metadata.getSchema().getType("f1_text");
+ Assertions.assertEquals(
+ PrimitiveType.PrimitiveTypeName.BINARY,
+ f1Type.asPrimitiveType().getPrimitiveTypeName());
+ Assertions.assertEquals(
+ LogicalTypeAnnotation.stringType(), f1Type.getLogicalTypeAnnotation());
+
+ Type f2Type = metadata.getSchema().getType("f2_timestamp");
+ Assertions.assertEquals(
+ PrimitiveType.PrimitiveTypeName.INT96,
+ f2Type.asPrimitiveType().getPrimitiveTypeName());
+ Type f3Type = metadata.getSchema().getType("f3_bytes");
+ Assertions.assertEquals(
+ PrimitiveType.PrimitiveTypeName.INT96,
+ f3Type.asPrimitiveType().getPrimitiveTypeName());
+ }
+
+ SeaTunnelRowType readRowType = readStrategy.getSeaTunnelRowTypeInfo(readFilePath);
+ Assertions.assertEquals(
+ BasicType.STRING_TYPE.getSqlType(), readRowType.getFieldType(0).getSqlType());
+ Assertions.assertEquals(
+ LocalTimeType.LOCAL_DATE_TIME_TYPE.getSqlType(),
+ readRowType.getFieldType(1).getSqlType());
+ Assertions.assertEquals(
+ LocalTimeType.LOCAL_DATE_TIME_TYPE.getSqlType(),
+ readRowType.getFieldType(2).getSqlType());
+ List<SeaTunnelRow> readRows = new ArrayList<>();
+ Collector<SeaTunnelRow> readCollector =
+ new Collector<SeaTunnelRow>() {
+ @Override
+ public void collect(SeaTunnelRow record) {
+ Assertions.assertTrue(record.getField(0) instanceof String);
+ Assertions.assertTrue(record.getField(1) instanceof LocalDateTime);
+ Assertions.assertTrue(record.getField(2) instanceof LocalDateTime);
+ readRows.add(record);
+ }
+
+ @Override
+ public Object getCheckpointLock() {
+ return null;
+ }
+ };
+ readStrategy.read(readFilePath, "test", readCollector);
+ Assertions.assertEquals(1, readRows.size());
+ readStrategy.close();
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-cos/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/cos/sink/CosFileSinkFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-cos/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/cos/sink/CosFileSinkFactory.java
index fb45c94717..e0f30ee907 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-cos/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/cos/sink/CosFileSinkFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-cos/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/cos/sink/CosFileSinkFactory.java
@@ -66,7 +66,9 @@ public class CosFileSinkFactory implements TableSinkFactory {
.conditional(
BaseSinkConfig.FILE_FORMAT_TYPE,
FileFormat.PARQUET,
- BaseSinkConfig.PARQUET_COMPRESS)
+ BaseSinkConfig.PARQUET_COMPRESS,
+ BaseSinkConfig.PARQUET_AVRO_WRITE_FIXED_AS_INT96,
+ BaseSinkConfig.PARQUET_AVRO_WRITE_TIMESTAMP_AS_INT96)
.conditional(
BaseSinkConfig.FILE_FORMAT_TYPE,
FileFormat.XML,
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-ftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/ftp/sink/FtpFileSinkFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-ftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/ftp/sink/FtpFileSinkFactory.java
index f3a12d117a..24a9ed48f8 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-ftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/ftp/sink/FtpFileSinkFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-ftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/ftp/sink/FtpFileSinkFactory.java
@@ -66,7 +66,9 @@ public class FtpFileSinkFactory implements TableSinkFactory {
.conditional(
BaseSinkConfig.FILE_FORMAT_TYPE,
FileFormat.PARQUET,
- BaseSinkConfig.PARQUET_COMPRESS)
+ BaseSinkConfig.PARQUET_COMPRESS,
+ BaseSinkConfig.PARQUET_AVRO_WRITE_FIXED_AS_INT96,
+ BaseSinkConfig.PARQUET_AVRO_WRITE_TIMESTAMP_AS_INT96)
.conditional(
BaseSinkConfig.FILE_FORMAT_TYPE,
FileFormat.XML,
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-hadoop/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/hdfs/sink/HdfsFileSinkFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-hadoop/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/hdfs/sink/HdfsFileSinkFactory.java
index fa2f872b0e..521456ca09 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-hadoop/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/hdfs/sink/HdfsFileSinkFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-hadoop/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/hdfs/sink/HdfsFileSinkFactory.java
@@ -63,7 +63,9 @@ public class HdfsFileSinkFactory implements TableSinkFactory {
.conditional(
BaseSinkConfig.FILE_FORMAT_TYPE,
FileFormat.PARQUET,
- BaseSinkConfig.PARQUET_COMPRESS)
+ BaseSinkConfig.PARQUET_COMPRESS,
+ BaseSinkConfig.PARQUET_AVRO_WRITE_FIXED_AS_INT96,
+ BaseSinkConfig.PARQUET_AVRO_WRITE_TIMESTAMP_AS_INT96)
.conditional(
BaseSinkConfig.FILE_FORMAT_TYPE,
FileFormat.XML,
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-jindo-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/jindo/sink/OssFileSinkFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-jindo-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/jindo/sink/OssFileSinkFactory.java
index b1bfb439e0..2f29fb19cd 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-jindo-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/jindo/sink/OssFileSinkFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-jindo-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/jindo/sink/OssFileSinkFactory.java
@@ -66,7 +66,9 @@ public class OssFileSinkFactory implements TableSinkFactory {
.conditional(
BaseSinkConfig.FILE_FORMAT_TYPE,
FileFormat.PARQUET,
- BaseSinkConfig.PARQUET_COMPRESS)
+ BaseSinkConfig.PARQUET_COMPRESS,
+ BaseSinkConfig.PARQUET_AVRO_WRITE_FIXED_AS_INT96,
+ BaseSinkConfig.PARQUET_AVRO_WRITE_TIMESTAMP_AS_INT96)
.conditional(
BaseSinkConfig.FILE_FORMAT_TYPE,
FileFormat.XML,
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-local/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/local/sink/LocalFileSinkFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-local/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/local/sink/LocalFileSinkFactory.java
index b2e744722f..01d5fb1060 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-local/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/local/sink/LocalFileSinkFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-local/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/local/sink/LocalFileSinkFactory.java
@@ -69,7 +69,9 @@ public class LocalFileSinkFactory extends BaseMultipleTableFileSinkFactory {
.conditional(
BaseSinkConfig.FILE_FORMAT_TYPE,
FileFormat.PARQUET,
- BaseSinkConfig.PARQUET_COMPRESS)
+ BaseSinkConfig.PARQUET_COMPRESS,
+ BaseSinkConfig.PARQUET_AVRO_WRITE_FIXED_AS_INT96,
+ BaseSinkConfig.PARQUET_AVRO_WRITE_TIMESTAMP_AS_INT96)
.conditional(
BaseSinkConfig.FILE_FORMAT_TYPE,
FileFormat.XML,
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/sink/OssFileSinkFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/sink/OssFileSinkFactory.java
index 38882175c7..edf6610714 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/sink/OssFileSinkFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/sink/OssFileSinkFactory.java
@@ -80,7 +80,9 @@ public class OssFileSinkFactory extends BaseMultipleTableFileSinkFactory {
.conditional(
BaseSinkConfig.FILE_FORMAT_TYPE,
FileFormat.PARQUET,
- BaseSinkConfig.PARQUET_COMPRESS)
+ BaseSinkConfig.PARQUET_COMPRESS,
+ BaseSinkConfig.PARQUET_AVRO_WRITE_FIXED_AS_INT96,
+ BaseSinkConfig.PARQUET_AVRO_WRITE_TIMESTAMP_AS_INT96)
.conditional(
BaseSinkConfig.FILE_FORMAT_TYPE,
FileFormat.XML,
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-s3/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/s3/sink/S3FileSinkFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-s3/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/s3/sink/S3FileSinkFactory.java
index 3a4e3df2fb..81ac767404 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-s3/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/s3/sink/S3FileSinkFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-s3/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/s3/sink/S3FileSinkFactory.java
@@ -85,7 +85,9 @@ public class S3FileSinkFactory implements TableSinkFactory {
.conditional(
BaseSinkConfig.FILE_FORMAT_TYPE,
FileFormat.PARQUET,
- BaseSinkConfig.PARQUET_COMPRESS)
+ BaseSinkConfig.PARQUET_COMPRESS,
+ BaseSinkConfig.PARQUET_AVRO_WRITE_FIXED_AS_INT96,
+ BaseSinkConfig.PARQUET_AVRO_WRITE_TIMESTAMP_AS_INT96)
.conditional(
BaseSinkConfig.FILE_FORMAT_TYPE,
FileFormat.XML,
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/sink/SftpFileSinkFactory.java b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/sink/SftpFileSinkFactory.java
index 2dc2c29bd9..e8116c35f1 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/sink/SftpFileSinkFactory.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/sink/SftpFileSinkFactory.java
@@ -66,7 +66,9 @@ public class SftpFileSinkFactory implements TableSinkFactory {
.conditional(
BaseSinkConfig.FILE_FORMAT_TYPE,
FileFormat.PARQUET,
- BaseSinkConfig.PARQUET_COMPRESS)
+ BaseSinkConfig.PARQUET_COMPRESS,
+ BaseSinkConfig.PARQUET_AVRO_WRITE_FIXED_AS_INT96,
+ BaseSinkConfig.PARQUET_AVRO_WRITE_TIMESTAMP_AS_INT96)
.conditional(
BaseSinkConfig.FILE_FORMAT_TYPE,
FileFormat.XML,