You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ty...@apache.org on 2023/05/17 15:56:53 UTC
[incubator-seatunnel] branch dev updated: [Hotfix][Connector-v2][File] Fix configuration file format and error comments (#4762)
This is an automated email from the ASF dual-hosted git repository.
tyrantlucifer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new a9002066a [Hotfix][Connector-v2][File] Fix configuration file format and error comments (#4762)
a9002066a is described below
commit a9002066a2047be54c822f9b5dcda580cc55d4c3
Author: monster <60...@users.noreply.github.com>
AuthorDate: Wed May 17 23:56:46 2023 +0800
[Hotfix][Connector-v2][File] Fix configuration file format and error comments (#4762)
---
.../test/resources/excel/fake_to_local_excel.conf | 17 ++++------
.../excel/local_excel_projection_to_assert.conf | 38 ++++++++++------------
.../resources/excel/local_excel_to_assert.conf | 38 ++++++++++------------
.../resources/excel/fakesource_to_sftp_excel.conf | 3 --
.../excel/sftp_excel_projection_to_assert.conf | 5 +--
.../test/resources/excel/sftp_excel_to_assert.conf | 4 ---
.../resources/json/fake_to_sftp_file_json.conf | 1 +
.../resources/text/fake_to_sftp_file_text.conf | 1 +
8 files changed, 44 insertions(+), 63 deletions(-)
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/excel/fake_to_local_excel.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/excel/fake_to_local_excel.conf
index ac7a01c91..ff72a2341 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/excel/fake_to_local_excel.conf
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/excel/fake_to_local_excel.conf
@@ -14,9 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-######
-###### This config file is a demonstration of streaming processing in seatunnel config
-######
env {
execution.parallelism = 1
@@ -70,12 +67,12 @@ source {
sink {
LocalFile {
- path="/tmp/seatunnel/excel"
- partition_dir_expression="${k0}=${v0}"
- is_partition_field_write_in_file=true
- file_name_expression="${transactionId}_${now}"
- file_format_type="excel"
- filename_time_format="yyyy.MM.dd"
- is_enable_transaction=true
+ path = "/tmp/seatunnel/excel"
+ partition_dir_expression = "${k0}=${v0}"
+ is_partition_field_write_in_file = true
+ file_name_expression = "${transactionId}_${now}"
+ file_format_type = "excel"
+ filename_time_format = "yyyy.MM.dd"
+ is_enable_transaction = true
}
}
\ No newline at end of file
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/excel/local_excel_projection_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/excel/local_excel_projection_to_assert.conf
index 17cd6d15e..df6749f71 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/excel/local_excel_projection_to_assert.conf
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/excel/local_excel_projection_to_assert.conf
@@ -14,9 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-######
-###### This config file is a demonstration of streaming processing in seatunnel config
-######
env {
execution.parallelism = 1
@@ -30,7 +27,7 @@ env {
source {
LocalFile {
- path="/seatunnel/read/excel"
+ path = "/seatunnel/read/excel"
result_table_name = "fake"
file_format_type = excel
delimiter = ;
@@ -38,6 +35,21 @@ source {
skip_header_row_number = 1
schema = {
fields {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ c_row = {
c_map = "map<string, string>"
c_array = "array<int>"
c_string = string
@@ -52,28 +64,12 @@ source {
c_date = date
c_decimal = "decimal(38, 18)"
c_timestamp = timestamp
- c_row = {
- c_map = "map<string, string>"
- c_array = "array<int>"
- c_string = string
- c_boolean = boolean
- c_tinyint = tinyint
- c_smallint = smallint
- c_int = int
- c_bigint = bigint
- c_float = float
- c_double = double
- c_bytes = bytes
- c_date = date
- c_decimal = "decimal(38, 18)"
- c_timestamp = timestamp
- }
+ }
}
}
}
}
-
sink {
Assert {
rules {
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/excel/local_excel_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/excel/local_excel_to_assert.conf
index 0f463aaf6..1160ac5f2 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/excel/local_excel_to_assert.conf
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/excel/local_excel_to_assert.conf
@@ -14,9 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-######
-###### This config file is a demonstration of streaming processing in seatunnel config
-######
env {
execution.parallelism = 1
@@ -30,13 +27,28 @@ env {
source {
LocalFile {
- path="/seatunnel/read/excel"
+ path = "/seatunnel/read/excel"
result_table_name = "fake"
file_format_type = excel
delimiter = ;
skip_header_row_number = 1
schema = {
fields {
+ c_map = "map<string, string>"
+ c_array = "array<int>"
+ c_string = string
+ c_boolean = boolean
+ c_tinyint = tinyint
+ c_smallint = smallint
+ c_int = int
+ c_bigint = bigint
+ c_float = float
+ c_double = double
+ c_bytes = bytes
+ c_date = date
+ c_decimal = "decimal(38, 18)"
+ c_timestamp = timestamp
+ c_row = {
c_map = "map<string, string>"
c_array = "array<int>"
c_string = string
@@ -51,28 +63,12 @@ source {
c_date = date
c_decimal = "decimal(38, 18)"
c_timestamp = timestamp
- c_row = {
- c_map = "map<string, string>"
- c_array = "array<int>"
- c_string = string
- c_boolean = boolean
- c_tinyint = tinyint
- c_smallint = smallint
- c_int = int
- c_bigint = bigint
- c_float = float
- c_double = double
- c_bytes = bytes
- c_date = date
- c_decimal = "decimal(38, 18)"
- c_timestamp = timestamp
- }
+ }
}
}
}
}
-
sink {
Assert {
rules {
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/excel/fakesource_to_sftp_excel.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/excel/fakesource_to_sftp_excel.conf
index c6c7b2e13..17a0f20f6 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/excel/fakesource_to_sftp_excel.conf
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/excel/fakesource_to_sftp_excel.conf
@@ -14,9 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-######
-###### This config file is a demonstration of streaming processing in seatunnel config
-######
env {
# You can set flink configuration here
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/excel/sftp_excel_projection_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/excel/sftp_excel_projection_to_assert.conf
index 32e13206d..356c0a811 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/excel/sftp_excel_projection_to_assert.conf
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/excel/sftp_excel_projection_to_assert.conf
@@ -14,9 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-######
-###### This config file is a demonstration of streaming processing in seatunnel config
-######
+
env {
# You can set flink configuration here
execution.parallelism = 1
@@ -79,7 +77,6 @@ source {
}
}
-
sink {
Assert {
source_table_name = "sftp"
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/excel/sftp_excel_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/excel/sftp_excel_to_assert.conf
index 3b9cc0527..0031b3209 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/excel/sftp_excel_to_assert.conf
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/excel/sftp_excel_to_assert.conf
@@ -14,9 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-######
-###### This config file is a demonstration of streaming processing in seatunnel config
-######
env {
# You can set flink configuration here
@@ -79,7 +76,6 @@ source {
}
}
-
sink {
Assert {
source_table_name = "sftp"
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/json/fake_to_sftp_file_json.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/json/fake_to_sftp_file_json.conf
index 9f9ae75b2..4e222d0c1 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/json/fake_to_sftp_file_json.conf
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/json/fake_to_sftp_file_json.conf
@@ -27,6 +27,7 @@ env {
spark.executor.memory = "1g"
spark.master = local
}
+
source {
FakeSource {
result_table_name = "sftp"
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/fake_to_sftp_file_text.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/fake_to_sftp_file_text.conf
index e17ff90af..0c0d69f0f 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/fake_to_sftp_file_text.conf
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/fake_to_sftp_file_text.conf
@@ -27,6 +27,7 @@ env {
spark.executor.memory = "1g"
spark.master = local
}
+
source {
FakeSource {
result_table_name = "sftp"