You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ki...@apache.org on 2022/05/16 12:43:43 UTC
[incubator-seatunnel] branch dev updated: [Feature][Flink-SQL-connector] add flink sql connector kafka and docs (#1878)
This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 6cade31e [Feature][Flink-SQL-connector] add flink sql connector kafka and docs (#1878)
6cade31e is described below
commit 6cade31e521fdd41589143edceb6fb6735f58ab3
Author: legendtkl <ta...@gmail.com>
AuthorDate: Mon May 16 20:43:38 2022 +0800
[Feature][Flink-SQL-connector] add flink sql connector kafka and docs (#1878)
* [Feature][Flink-SQL-connector] add flink sql connector kafka and docs
* address review comment
Co-authored-by: ruanwenjun <we...@apache.org>
---
docs/en/connector/flink-sql/Kafka.md | 74 ++++++++++++++++++++++
pom.xml | 19 ++++++
.../seatunnel-connectors-flink-sql-dist/pom.xml | 6 ++
.../{ => flink-sql-connector-kafka}/pom.xml | 26 ++++----
.../seatunnel-connectors-flink-sql/pom.xml | 1 +
seatunnel-dist/release-docs/LICENSE | 12 ----
seatunnel-e2e/seatunnel-flink-e2e/pom.xml | 1 -
seatunnel-e2e/seatunnel-flink-sql-e2e/pom.xml | 1 -
seatunnel-e2e/seatunnel-spark-e2e/pom.xml | 1 -
.../seatunnel-flink-examples/pom.xml | 1 -
.../seatunnel-flink-sql-examples/pom.xml | 1 -
.../seatunnel-spark-examples/pom.xml | 1 -
tools/dependencies/known-dependencies.txt | 7 --
13 files changed, 114 insertions(+), 37 deletions(-)
diff --git a/docs/en/connector/flink-sql/Kafka.md b/docs/en/connector/flink-sql/Kafka.md
new file mode 100644
index 00000000..a24a0a60
--- /dev/null
+++ b/docs/en/connector/flink-sql/Kafka.md
@@ -0,0 +1,74 @@
+# Flink SQL Kafka Connector
+
+## Description
+
+With kafka connector, we can read data from kafka and write data to kafka using Flink SQL. Refer to the [Kafka connector](https://nightlies.apache.org/flink/flink-docs-release-1.13/docs/connectors/table/kafka/) for more details.
+
+
+## Usage
+Let us have a brief example to show how to use the connector from end to end.
+
+### 1. kafka prepare
+Please refer to the [Kafka QuickStart](https://kafka.apache.org/quickstart) to prepare kafka environment and produce data like following:
+
+```bash
+$ bin/kafka-console-producer.sh --topic <topic-name> --bootstrap-server localhost:9092
+```
+
+After executing the command, we will come to the interactive mode. Print the following message to send data to kafka.
+```bash
+{"id":1,"name":"abc"}
+>{"id":2,"name":"def"}
+>{"id":3,"name":"dfs"}
+>{"id":4,"name":"eret"}
+>{"id":5,"name":"yui"}
+```
+
+### 2. prepare seatunnel configuration
+Here is a simple example of seatunnel configuration.
+```sql
+SET table.dml-sync = true;
+
+CREATE TABLE events (
+ id INT,
+ name STRING
+) WITH (
+ 'connector' = 'kafka',
+ 'topic'='<topic-name>',
+ 'properties.bootstrap.servers' = 'localhost:9092',
+ 'properties.group.id' = 'testGroup',
+ 'scan.startup.mode' = 'earliest-offset',
+ 'format' = 'json'
+);
+
+CREATE TABLE print_table (
+ id INT,
+ name STRING
+) WITH (
+ 'connector' = 'print',
+ 'sink.parallelism' = '1'
+);
+
+INSERT INTO print_table SELECT * FROM events;
+```
+
+### 3. start flink local cluster
+```bash
+$ ${FLINK_HOME}/bin/start-cluster.sh
+```
+
+### 4. start Flink SQL job
+Execute the following command in seatunnel home path to start the Flink SQL job.
+```bash
+$ bin/start-seatunnel-sql.sh -c config/kafka.sql.conf
+```
+
+### 5. verify result
+After the job submitted, we can see the data printing by connector 'print' in taskmanager's log .
+```text
++I[1, abc]
++I[2, def]
++I[3, dfs]
++I[4, eret]
++I[5, yui]
+```
diff --git a/pom.xml b/pom.xml
index 92d09a96..90c6483d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -173,6 +173,7 @@
<guava.version>19.0</guava.version>
<auto-service.version>1.0.1</auto-service.version>
<jmockdata.version>4.3.0</jmockdata.version>
+ <snappy-java.version>1.1.8.3</snappy-java.version>
</properties>
<dependencyManagement>
@@ -611,6 +612,24 @@
<artifactId>jmockdata</artifactId>
<version>${jmockdata.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ <version>${snappy-java.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
diff --git a/seatunnel-connectors/seatunnel-connectors-flink-sql-dist/pom.xml b/seatunnel-connectors/seatunnel-connectors-flink-sql-dist/pom.xml
index a5b26eac..1310c5c5 100644
--- a/seatunnel-connectors/seatunnel-connectors-flink-sql-dist/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-flink-sql-dist/pom.xml
@@ -31,6 +31,12 @@
<artifactId>flink-sql-connector-jdbc</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>flink-sql-connector-kafka</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
<build>
diff --git a/seatunnel-connectors/seatunnel-connectors-flink-sql/pom.xml b/seatunnel-connectors/seatunnel-connectors-flink-sql/flink-sql-connector-kafka/pom.xml
similarity index 68%
copy from seatunnel-connectors/seatunnel-connectors-flink-sql/pom.xml
copy to seatunnel-connectors/seatunnel-connectors-flink-sql/flink-sql-connector-kafka/pom.xml
index 79be2c28..ac518a75 100644
--- a/seatunnel-connectors/seatunnel-connectors-flink-sql/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-flink-sql/flink-sql-connector-kafka/pom.xml
@@ -16,19 +16,21 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>seatunnel-connectors</artifactId>
- <groupId>org.apache.seatunnel</groupId>
- <version>${revision}</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>seatunnel-connectors-flink-sql</artifactId>
+ <groupId>org.apache.seatunnel</groupId>
+ <version>${revision}</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
- <artifactId>seatunnel-connectors-flink-sql</artifactId>
- <packaging>pom</packaging>
+ <artifactId>flink-sql-connector-kafka</artifactId>
- <modules>
- <module>flink-sql-connector-jdbc</module>
- </modules>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
+ </dependency>
+ </dependencies>
-</project>
\ No newline at end of file
+</project>
diff --git a/seatunnel-connectors/seatunnel-connectors-flink-sql/pom.xml b/seatunnel-connectors/seatunnel-connectors-flink-sql/pom.xml
index 79be2c28..a23fccad 100644
--- a/seatunnel-connectors/seatunnel-connectors-flink-sql/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-flink-sql/pom.xml
@@ -28,6 +28,7 @@
<modules>
<module>flink-sql-connector-jdbc</module>
+ <module>flink-sql-connector-kafka</module>
</modules>
diff --git a/seatunnel-dist/release-docs/LICENSE b/seatunnel-dist/release-docs/LICENSE
index 15bb80df..9471de2a 100644
--- a/seatunnel-dist/release-docs/LICENSE
+++ b/seatunnel-dist/release-docs/LICENSE
@@ -816,7 +816,6 @@ The text of each license is the standard Apache 2.0 license.
(The Apache Software License, Version 2.0) Phoenix Core (org.apache.phoenix:phoenix-core:5.0.0-HBase-2.0 - http://www.apache.org/phoenix/phoenix-core/)
(The Apache Software License, Version 2.0) Plexus Interpolation API (org.codehaus.plexus:plexus-interpolation:1.19 - http://plexus.codehaus.org/plexus-components/plexus-interpolation)
(The Apache Software License, Version 2.0) Retrofit (com.squareup.retrofit2:retrofit:2.9.0 - https://github.com/square/retrofit)
- (The Apache Software License, Version 2.0) Snappy for Java (org.xerial.snappy:snappy-java:1.0.5 - http://github.com/xerial/snappy-java/)
(The Apache Software License, Version 2.0) SparseBitSet (com.zaxxer:SparseBitSet:1.2 - https://github.com/brettwooldridge/SparseBitSet)
(The Apache Software License, Version 2.0) Spymemcached (net.spy:spymemcached:2.12.3 - http://www.couchbase.org/code/couchbase/java)
(The Apache Software License, Version 2.0) StAX API (stax:stax-api:1.0.1 - http://stax.codehaus.org/)
@@ -884,10 +883,6 @@ The text of each license is the standard Apache 2.0 license.
(The Apache Software License, Version 2.0) secure-sm (org.elasticsearch:elasticsearch-secure-sm:6.3.1 - https://github.com/elastic/elasticsearch)
(The Apache Software License, Version 2.0) server (org.elasticsearch:elasticsearch:6.3.1 - https://github.com/elastic/elasticsearch)
(The Apache Software License, Version 2.0) server (org.elasticsearch:elasticsearch:7.5.1 - https://github.com/elastic/elasticsearch)
- (The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.2.6 - https://github.com/xerial/snappy-java)
- (The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.4 - https://github.com/xerial/snappy-java)
- (The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.7.1 - https://github.com/xerial/snappy-java)
- (The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.7.3 - https://github.com/xerial/snappy-java)
(The Apache Software License, Version 2.0) transport (org.elasticsearch.client:transport:6.3.1 - https://github.com/elastic/elasticsearch)
(The Apache Software License, Version 2.0) transport (org.elasticsearch.client:transport:7.5.1 - https://github.com/elastic/elasticsearch)
(The Apache Software License, Version 2.0) transport-netty4 (org.elasticsearch.plugin:transport-netty4-client:6.3.1 - https://github.com/elastic/elasticsearch)
@@ -911,14 +906,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
(MIT License) Joni (org.jruby.joni:joni:2.1.11 - http://nexus.sonatype.org/oss-repository-hosting.html/joni)
(MIT License) Joni (org.jruby.joni:joni:2.1.2 - http://nexus.sonatype.org/oss-repository-hosting.html/joni)
(MIT License) Joni (org.jruby.joni:joni:2.1.27 - http://nexus.sonatype.org/oss-repository-hosting.html/joni)
- (MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.6.4 - http://www.slf4j.org)
- (MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.7.15 - http://www.slf4j.org)
- (MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.7.16 - http://www.slf4j.org)
(MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.7.25 - http://www.slf4j.org)
- (MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.7.30 - http://www.slf4j.org)
- (MIT License) SLF4J LOG4J-12 Binding (org.slf4j:slf4j-log4j12:1.7.10 - http://www.slf4j.org)
- (MIT License) SLF4J LOG4J-12 Binding (org.slf4j:slf4j-log4j12:1.7.15 - http://www.slf4j.org)
- (MIT License) SLF4J LOG4J-12 Binding (org.slf4j:slf4j-log4j12:1.7.16 - http://www.slf4j.org)
(MIT License) SLF4J LOG4J-12 Binding (org.slf4j:slf4j-log4j12:1.7.25 - http://www.slf4j.org)
(MIT License) pyrolite (net.razorvine:pyrolite:4.13 - https://github.com/irmen/Pyrolite)
(MIT License) scopt (com.github.scopt:scopt_2.11:3.5.0 - https://github.com/scopt/scopt)
diff --git a/seatunnel-e2e/seatunnel-flink-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-e2e/pom.xml
index 04b992b9..a0651cc1 100644
--- a/seatunnel-e2e/seatunnel-flink-e2e/pom.xml
+++ b/seatunnel-e2e/seatunnel-flink-e2e/pom.xml
@@ -36,7 +36,6 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
<scope>test</scope>
</dependency>
diff --git a/seatunnel-e2e/seatunnel-flink-sql-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-sql-e2e/pom.xml
index 284c138e..e8cdd839 100644
--- a/seatunnel-e2e/seatunnel-flink-sql-e2e/pom.xml
+++ b/seatunnel-e2e/seatunnel-flink-sql-e2e/pom.xml
@@ -35,7 +35,6 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
- <version>1.7.25</version>
<scope>test</scope>
</dependency>
diff --git a/seatunnel-e2e/seatunnel-spark-e2e/pom.xml b/seatunnel-e2e/seatunnel-spark-e2e/pom.xml
index a613ca93..e6a57972 100644
--- a/seatunnel-e2e/seatunnel-spark-e2e/pom.xml
+++ b/seatunnel-e2e/seatunnel-spark-e2e/pom.xml
@@ -30,7 +30,6 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/seatunnel-examples/seatunnel-flink-examples/pom.xml b/seatunnel-examples/seatunnel-flink-examples/pom.xml
index 79fbe8f1..d2dadebc 100644
--- a/seatunnel-examples/seatunnel-flink-examples/pom.xml
+++ b/seatunnel-examples/seatunnel-flink-examples/pom.xml
@@ -93,7 +93,6 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
</dependency>
</dependencies>
diff --git a/seatunnel-examples/seatunnel-flink-sql-examples/pom.xml b/seatunnel-examples/seatunnel-flink-sql-examples/pom.xml
index 63a714fd..bed720dc 100644
--- a/seatunnel-examples/seatunnel-flink-sql-examples/pom.xml
+++ b/seatunnel-examples/seatunnel-flink-sql-examples/pom.xml
@@ -94,7 +94,6 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
</dependency>
</dependencies>
</project>
diff --git a/seatunnel-examples/seatunnel-spark-examples/pom.xml b/seatunnel-examples/seatunnel-spark-examples/pom.xml
index 6b9d2c79..757aed3d 100644
--- a/seatunnel-examples/seatunnel-spark-examples/pom.xml
+++ b/seatunnel-examples/seatunnel-spark-examples/pom.xml
@@ -88,7 +88,6 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt
index 246adc0f..631e319c 100755
--- a/tools/dependencies/known-dependencies.txt
+++ b/tools/dependencies/known-dependencies.txt
@@ -633,18 +633,11 @@ shims-0.9.0.jar
shims-0.9.22.jar
sigar-1.6.5.132.jar
sketches-core-0.9.0.jar
-slf4j-api-1.7.15.jar
-slf4j-api-1.7.16.jar
-slf4j-api-1.7.21.jar
slf4j-api-1.7.25.jar
-slf4j-log4j12-1.7.15.jar
-slf4j-log4j12-1.7.16.jar
slf4j-log4j12-1.7.25.jar
snakeyaml-1.17.jar
snakeyaml-1.24.jar
snappy-0.3.jar
-snappy-java-1.1.4.jar
-snappy-java-1.1.7.1.jar
snappy-java-1.1.8.3.jar
spark-catalyst_2.11-2.4.0.jar
spark-hive-thriftserver_2.11-2.3.4.jar