You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by we...@apache.org on 2022/04/21 05:16:23 UTC
[incubator-seatunnel] branch dev updated: [Improve][Connectors]add IT tetscase for http soure (#1718)
This is an automated email from the ASF dual-hosted git repository.
wenjun pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new e5613b20 [Improve][Connectors]add IT tetscase for http soure (#1718)
e5613b20 is described below
commit e5613b20af032a11db4b9ee4fcccb7f70d8f79db
Author: tmljob <69...@users.noreply.github.com>
AuthorDate: Thu Apr 21 13:16:18 2022 +0800
[Improve][Connectors]add IT tetscase for http soure (#1718)
* add IT test case for Http Soure
---
seatunnel-core/seatunnel-core-spark/pom.xml | 6 ++
.../e2e/spark/fake/HttpSourceToConsoleIT.java | 40 ++++++++++++
.../test/resources/http/httpsource_to_console.conf | 73 ++++++++++++++++++++++
3 files changed, 119 insertions(+)
diff --git a/seatunnel-core/seatunnel-core-spark/pom.xml b/seatunnel-core/seatunnel-core-spark/pom.xml
index 2492f329..aae0a748 100644
--- a/seatunnel-core/seatunnel-core-spark/pom.xml
+++ b/seatunnel-core/seatunnel-core-spark/pom.xml
@@ -177,6 +177,12 @@
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-connector-spark-http</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-transform-spark-split</artifactId>
diff --git a/seatunnel-e2e/seatunnel-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/fake/HttpSourceToConsoleIT.java b/seatunnel-e2e/seatunnel-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/fake/HttpSourceToConsoleIT.java
new file mode 100644
index 00000000..0f4bdcfb
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/fake/HttpSourceToConsoleIT.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.e2e.spark.fake;
+
+import org.apache.seatunnel.e2e.spark.SparkContainer;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.testcontainers.containers.Container;
+
+import java.io.IOException;
+
+/**
+ * This test case is used to verify that the http source is able to send data to the console.
+ * Make sure the SeaTunnel job can submit successfully on spark engine.
+ */
+public class HttpSourceToConsoleIT extends SparkContainer {
+
+ @Test
+ public void testHttpSourceToConsoleSine() throws IOException, InterruptedException {
+ Container.ExecResult execResult = executeSeaTunnelSparkJob("/http/httpsource_to_console.conf");
+ Assert.assertEquals(0, execResult.getExitCode());
+ }
+
+}
diff --git a/seatunnel-e2e/seatunnel-spark-e2e/src/test/resources/http/httpsource_to_console.conf b/seatunnel-e2e/seatunnel-spark-e2e/src/test/resources/http/httpsource_to_console.conf
new file mode 100644
index 00000000..9b75e524
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-spark-e2e/src/test/resources/http/httpsource_to_console.conf
@@ -0,0 +1,73 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+######
+###### This config file is a demonstration of batch processing in seatunnel config
+######
+
+env {
+ # You can set spark configuration here
+ # see available properties defined by spark: https://spark.apache.org/docs/latest/configuration.html#available-properties
+ spark.app.name = "SeaTunnel"
+ spark.executor.instances = 2
+ spark.executor.cores = 1
+ spark.executor.memory = "1g"
+ spark.master = local
+}
+
+source {
+ # This is a example input plugin **only for test and demonstrate the feature input plugin**
+ Http {
+ url = "http://date.jsontest.com/"
+ result_table_name= "response_body"
+ }
+
+ # You can also use other input plugins, such as hdfs
+ # hdfs {
+ # result_table_name = "accesslog"
+ # path = "hdfs://hadoop-cluster-01/nginx/accesslog"
+ # format = "json"
+ # }
+
+ # If you would like to get more information about how to configure seatunnel and see full list of input plugins,
+ # please go to https://seatunnel.apache.org/docs/spark/configuration/source-plugins/Fake
+}
+
+transform {
+ # split data by specific delimiter
+
+ # you can also use other transform plugins, such as sql
+ # sql {
+ # sql = "select * from accesslog where request_time > 1000"
+ # }
+
+ # If you would like to get more information about how to configure seatunnel and see full list of transform plugins,
+ # please go to https://seatunnel.apache.org/docs/spark/configuration/transform-plugins/Split
+}
+
+sink {
+ # choose stdout output plugin to output data to console
+ Console {}
+
+ # you can also you other output plugins, such as sql
+ # hdfs {
+ # path = "hdfs://hadoop-cluster-01/nginx/accesslog_processed"
+ # save_mode = "append"
+ # }
+
+ # If you would like to get more information about how to configure seatunnel and see full list of output plugins,
+ # please go to https://seatunnel.apache.org/docs/spark/configuration/sink-plugins/Console
+}
\ No newline at end of file