[incubator-seatunnel] branch dev updated: [Feature][Connector-V2] Add http sink(Webhook) (#2348)

[fa...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

fanjia pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git


The following commit(s) were added to refs/heads/dev by this push:
     new 4b7207490 [Feature][Connector-V2] Add http sink(Webhook) (#2348)
4b7207490 is described below

commit 4b7207490a8613f94b2574daef93051e62151383
Author: TyrantLucifer <Ty...@gmail.com>
AuthorDate: Wed Aug 3 18:09:21 2022 +0800

    [Feature][Connector-V2] Add http sink(Webhook) (#2348)
    
    * [Feature][Connector-V2] Add first version of http sink (#2110)
    
    * [Improve][Connector-V2] Http client provider improve (#2312)
    
    * [Improve][Connector-V2] Fix 'Singleton' word error
    
    * [Improve][Connector-V2]: Improve http client provider code structure
    
    1. Remove hard code
    2. Http post method support request with json body
    
    * [Improve][Http Connector-V2-Source] Refactor the code and make code more clearly
    
    1. Unified http parameter for http sink and source
    2. Change http plugin
    config option constant class name from 'Config' to 'HttpConfig' to avoid
    conflict with typesafe config
    
    * [Improve][Http Connector-V2-Source] Update plugin name from 'Http' to 'HttpSource' and update doc
    
    * [Feature][Http Connector-V2-Sink] Add http sink
    
    * [Feature][Http Connector-V2-Sink] Add http sink doc
    
    * [Revert][Http Connector-V2-Source] In order to unified connector's name, so revert plugin name and doc
    
    * [Feature][Http Connector-V2-Sink] Rename http sink plugin name from 'HttpSink' to 'Http'
    
    * [Feature][Http Connector-V2-Sink] Add http sink in plugin-mapping.properties
    
    * [Feature][Http Connector-V2-Sink] Remove redundant import
---
 docs/en/connector-v2/sink/Http.md                  | 38 ++++++++++
 plugin-mapping.properties                          |  1 +
 .../connectors/seatunnel/http/sink/HttpSink.java   | 83 ++++++++++++++++++++++
 .../seatunnel/http/sink/HttpSinkWriter.java        | 73 +++++++++++++++++++
 4 files changed, 195 insertions(+)

diff --git a/docs/en/connector-v2/sink/Http.md b/docs/en/connector-v2/sink/Http.md
new file mode 100644
index 000000000..a83d4a679
--- /dev/null
+++ b/docs/en/connector-v2/sink/Http.md
@@ -0,0 +1,38 @@
+# Http
+
+## Description
+
+Used to launch web hooks using data. Both support streaming and batch mode.
+
+> For example, if the data from upstream is [`age: 12, name: tyrantlucifer`], the body content is the following: `{"age": 12, "name": "tyrantlucifer"}`
+
+**Tips: Http sink only support `post json` webhook and the data from source will be treated as body content in web hook.**
+
+##  Options
+
+| name | type   | required | default value |
+| --- |--------| --- | --- |
+| url | String | Yes | - |
+| headers | Map    | No | - |
+
+### url [string]
+
+http request url
+
+### headers [Map]
+
+http headers
+
+## Example
+
+simple:
+
+```hocon
+Http {
+        url = "http://localhost/test/webhook"
+        headers {
+            token = "9e32e859ef044462a257e1fc76730066"
+        }
+    }
+```
+
diff --git a/plugin-mapping.properties b/plugin-mapping.properties
index 5bc85ec72..c448b86bc 100644
--- a/plugin-mapping.properties
+++ b/plugin-mapping.properties
@@ -94,6 +94,7 @@ seatunnel.sink.Assert = connector-assert
 seatunnel.source.Kafka = connector-kafka
 seatunnel.sink.Kafka = connector-kafka
 seatunnel.source.Http = connector-http
+seatunnel.sink.Http = connector-http
 seatunnel.source.Socket = connector-socket
 seatunnel.sink.Hive = connector-hive
 seatunnel.source.Hive = connector-hive
diff --git a/seatunnel-connectors-v2/connector-http/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/sink/HttpSink.java b/seatunnel-connectors-v2/connector-http/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/sink/HttpSink.java
new file mode 100644
index 000000000..8466b8b2a
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-http/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/sink/HttpSink.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.http.sink;
+
+import org.apache.seatunnel.api.common.PrepareFailException;
+import org.apache.seatunnel.api.sink.SeaTunnelSink;
+import org.apache.seatunnel.api.sink.SinkWriter;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.common.config.CheckConfigUtil;
+import org.apache.seatunnel.common.config.CheckResult;
+import org.apache.seatunnel.common.constants.PluginType;
+import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSimpleSink;
+import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSinkWriter;
+import org.apache.seatunnel.connectors.seatunnel.http.config.HttpConfig;
+import org.apache.seatunnel.connectors.seatunnel.http.config.HttpParameter;
+
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+
+import com.google.auto.service.AutoService;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@AutoService(SeaTunnelSink.class)
+public class HttpSink extends AbstractSimpleSink<SeaTunnelRow, Void> {
+    private final HttpParameter httpParameter = new HttpParameter();
+    private SeaTunnelRowType seaTunnelRowType;
+    private Config pluginConfig;
+
+    @Override
+    public String getPluginName() {
+        return "Http";
+    }
+
+    @Override
+    public void prepare(Config pluginConfig) throws PrepareFailException {
+        this.pluginConfig = pluginConfig;
+        CheckResult result = CheckConfigUtil.checkAllExists(pluginConfig, HttpConfig.URL);
+        if (!result.isSuccess()) {
+            throw new PrepareFailException(getPluginName(), PluginType.SINK, result.getMsg());
+        }
+        httpParameter.setUrl(pluginConfig.getString(HttpConfig.URL));
+        if (pluginConfig.hasPath(HttpConfig.HEADERS)) {
+            httpParameter.setHeaders(pluginConfig.getConfig(HttpConfig.HEADERS).entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> String.valueOf(entry.getValue().unwrapped()), (v1, v2) -> v2)));
+        }
+        if (pluginConfig.hasPath(HttpConfig.PARAMS)) {
+            httpParameter.setHeaders(pluginConfig.getConfig(HttpConfig.PARAMS).entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> String.valueOf(entry.getValue().unwrapped()), (v1, v2) -> v2)));
+        }
+    }
+
+    @Override
+    public void setTypeInfo(SeaTunnelRowType seaTunnelRowType) {
+        this.seaTunnelRowType = seaTunnelRowType;
+    }
+
+    @Override
+    public SeaTunnelDataType<SeaTunnelRow> getConsumedType() {
+        return this.seaTunnelRowType;
+    }
+
+    @Override
+    public AbstractSinkWriter<SeaTunnelRow, Void> createWriter(SinkWriter.Context context) throws IOException {
+        return new HttpSinkWriter(seaTunnelRowType, httpParameter);
+    }
+}
diff --git a/seatunnel-connectors-v2/connector-http/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/sink/HttpSinkWriter.java b/seatunnel-connectors-v2/connector-http/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/sink/HttpSinkWriter.java
new file mode 100644
index 000000000..dfbc6576a
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-http/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/sink/HttpSinkWriter.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.http.sink;
+
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSinkWriter;
+import org.apache.seatunnel.connectors.seatunnel.http.client.HttpClientProvider;
+import org.apache.seatunnel.connectors.seatunnel.http.client.HttpResponse;
+import org.apache.seatunnel.connectors.seatunnel.http.config.HttpParameter;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Objects;
+
+public class HttpSinkWriter extends AbstractSinkWriter<SeaTunnelRow, Void> {
+    private static final Logger LOGGER = LoggerFactory.getLogger(HttpSinkWriter.class);
+    private final HttpClientProvider httpClient = HttpClientProvider.getInstance();
+    private final SeaTunnelRowType seaTunnelRowType;
+    private final HttpParameter httpParameter;
+
+    public HttpSinkWriter(SeaTunnelRowType seaTunnelRowType, HttpParameter httpParameter) {
+        this.seaTunnelRowType = seaTunnelRowType;
+        this.httpParameter = httpParameter;
+    }
+
+    @Override
+    public void write(SeaTunnelRow element) throws IOException {
+        ObjectMapper objectMapper = new ObjectMapper();
+        HashMap<Object, Object> objectMap = new HashMap<>();
+        int totalFields = seaTunnelRowType.getTotalFields();
+        for (int i = 0; i < totalFields; i++) {
+            objectMap.put(seaTunnelRowType.getFieldName(i), element.getField(i));
+        }
+        String body = objectMapper.writeValueAsString(objectMap);
+        try {
+            // only support post web hook
+            HttpResponse response = httpClient.doPost(httpParameter.getUrl(), httpParameter.getHeaders(), body);
+            if (HttpResponse.STATUS_OK == response.getCode()) {
+                return;
+            }
+            LOGGER.error("http client execute exception, http response status code:[{}], content:[{}]", response.getCode(), response.getContent());
+        } catch (Exception e) {
+            LOGGER.error(e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        if (Objects.nonNull(httpClient)) {
+            httpClient.close();
+        }
+    }
+}