You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by wa...@apache.org on 2023/12/02 04:27:30 UTC
(seatunnel) branch dev updated: [BUG][Connector-V2][Http] fix bug http config no schema option and improve e2e test add case (#5939)
This is an automated email from the ASF dual-hosted git repository.
wanghailin pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 8a71b9e072 [BUG][Connector-V2][Http] fix bug http config no schema option and improve e2e test add case (#5939)
8a71b9e072 is described below
commit 8a71b9e0726c14e85859ec86edd9cf85ab7f80f9
Author: lizhenglei <12...@users.noreply.github.com>
AuthorDate: Sat Dec 2 12:27:25 2023 +0800
[BUG][Connector-V2][Http] fix bug http config no schema option and improve e2e test add case (#5939)
---
docs/en/connector-v2/source/Http.md | 2 +-
.../seatunnel/http/config/HttpConfig.java | 5 +--
.../seatunnel/http/source/HttpSource.java | 13 +------
.../seatunnel/e2e/connector/http/HttpIT.java | 3 ++
.../src/test/resources/httpnoschema_to_http.conf | 40 ++++++++++++++++++++++
.../src/test/resources/mockserver-config.json | 29 ++++++++++++++++
6 files changed, 77 insertions(+), 15 deletions(-)
diff --git a/docs/en/connector-v2/source/Http.md b/docs/en/connector-v2/source/Http.md
index 0c01be813e..a1a1b2e7ec 100644
--- a/docs/en/connector-v2/source/Http.md
+++ b/docs/en/connector-v2/source/Http.md
@@ -53,7 +53,7 @@ They can be downloaded via install-plugin.sh or from the Maven central repositor
| pageing.total_page_size | Int | No | - | This parameter is used to control the total number of pages |
| pageing.batch_size | Int | No | - | The batch size returned per request is used to determine whether to continue when the total number of pages is unknown |
| content_json | String | No | - | This parameter can get some json data.If you only need the data in the 'book' section, configure `content_field = "$.store.book.*"`. |
-| format | String | No | json | The format of upstream data, now only support `json` `text`, default `json`. |
+| format | String | No | text | The format of upstream data, now only support `json` `text`, default `text`. |
| method | String | No | get | Http request method, only supports GET, POST method. |
| headers | Map | No | - | Http headers. |
| params | Map | No | - | Http params,the program will automatically add http header application/x-www-form-urlencoded. |
diff --git a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/config/HttpConfig.java b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/config/HttpConfig.java
index de21b73c7c..5db4c40e97 100644
--- a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/config/HttpConfig.java
+++ b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/config/HttpConfig.java
@@ -67,7 +67,7 @@ public class HttpConfig {
public static final Option<ResponseFormat> FORMAT =
Options.key("format")
.enumType(ResponseFormat.class)
- .defaultValue(ResponseFormat.JSON)
+ .defaultValue(ResponseFormat.TEXT)
.withDescription("Http response format");
public static final Option<Integer> POLL_INTERVAL_MILLS =
Options.key("poll_interval_millis")
@@ -113,7 +113,8 @@ public class HttpConfig {
"SeaTunnel enableMultiLines.This parameter can support http splitting response text by line.");
public enum ResponseFormat {
- JSON("json");
+ JSON("json"),
+ TEXT("text");
private String format;
diff --git a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSource.java b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSource.java
index 314deded06..34402e300e 100644
--- a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSource.java
+++ b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSource.java
@@ -31,9 +31,7 @@ import org.apache.seatunnel.api.table.catalog.TableIdentifier;
import org.apache.seatunnel.api.table.catalog.TableSchema;
import org.apache.seatunnel.api.table.catalog.schema.TableSchemaOptions;
import org.apache.seatunnel.api.table.type.BasicType;
-import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
-import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
import org.apache.seatunnel.common.config.CheckConfigUtil;
import org.apache.seatunnel.common.config.CheckResult;
import org.apache.seatunnel.common.constants.JobMode;
@@ -156,16 +154,7 @@ public class HttpSource extends AbstractSingleSplitSource<SeaTunnelRow> {
TableSchema.builder()
.column(
PhysicalColumn.of(
- "content",
- new SeaTunnelRowType(
- new String[] {"content"},
- new SeaTunnelDataType<?>[] {
- BasicType.STRING_TYPE
- }),
- 0,
- false,
- null,
- null))
+ "content", BasicType.STRING_TYPE, 0, false, null, null))
.build();
this.catalogTable =
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpIT.java
index 6266905a7b..405bc5157f 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpIT.java
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpIT.java
@@ -157,6 +157,9 @@ public class HttpIT extends TestSuiteBase implements TestResource {
Container.ExecResult execResult17 =
container.executeJob("/http_jsonrequestbody_to_feishu.conf");
Assertions.assertEquals(0, execResult17.getExitCode());
+
+ Container.ExecResult execResult18 = container.executeJob("/httpnoschema_to_http.conf");
+ Assertions.assertEquals(0, execResult18.getExitCode());
}
public String getMockServerConfig() {
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/httpnoschema_to_http.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/httpnoschema_to_http.conf
new file mode 100644
index 0000000000..6132493ce6
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/httpnoschema_to_http.conf
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ execution.parallelism = 1
+ job.mode = "BATCH"
+}
+
+source {
+ Http {
+ url = "http://mockserver:1080/example/jsonBody"
+ method = "POST"
+ body="{"id":1}"
+
+ }
+}
+
+sink {
+ Http {
+ url = "http://mockserver:1080/example/httpContentSink"
+ headers {
+ token = "9e32e859ef044462a257e1fc76730066"
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-config.json b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-config.json
index d41975d133..4ce23c4acb 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-config.json
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-config.json
@@ -4691,5 +4691,34 @@
"Content-Type": "application/json"
}
}
+ },
+ {
+ "httpRequest": {
+ "path": "/example/httpContentSink",
+ "method": "POST",
+ "headers": {
+ "token": ["9e32e859ef044462a257e1fc76730066"]
+ },
+ "body": {
+ "type": "JSON",
+ "json": {"content" : "[ {\n \"name\" : \"lzl\",\n \"age\" : 18\n}, {\n \"name\" : \"pizz\",\n \"age\" : 19\n} ]"},
+ "matchType": "STRICT"
+ }
+ },
+ "httpResponse": {
+ "body": [
+ {
+ "name": "lzl2",
+ "age": 18
+ },
+ {
+ "name": "pizz2",
+ "age": 19
+ }
+ ],
+ "headers": {
+ "Content-Type": "application/json"
+ }
+ }
}
]