You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ki...@apache.org on 2022/01/07 03:27:19 UTC
[incubator-seatunnel] branch dev updated: [Feature][seatunnel-examples] flink local environment run quickly and debug locally developed code easily (#960)
This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 950d9f4 [Feature][seatunnel-examples] flink local environment run quickly and debug locally developed code easily (#960)
950d9f4 is described below
commit 950d9f41f6fac16e3efc7934efec3df7a07c58ee
Author: felix.wang <59...@users.noreply.github.com>
AuthorDate: Fri Jan 7 11:27:12 2022 +0800
[Feature][seatunnel-examples] flink local environment run quickly and debug locally developed code easily (#960)
---
pom.xml | 1 +
seatunnel-examples/pom.xml | 35 +++++++++++
.../seatunnel-flink-examples/pom.xml | 70 ++++++++++++++++++++++
.../org/apache/seatunnel/test/flink/LocalTest.java | 38 ++++++++++++
.../main/resources/examples/fake_to_console.config | 55 +++++++++++++++++
.../src/main/resources/log4j.properties | 22 +++++++
6 files changed, 221 insertions(+)
diff --git a/pom.xml b/pom.xml
index 3ea2cad..9ccf8be 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,6 +57,7 @@
<module>seatunnel-transforms</module>
<module>seatunnel-connectors</module>
<module>seatunnel-dist</module>
+ <module>seatunnel-examples</module>
</modules>
<properties>
diff --git a/seatunnel-examples/pom.xml b/seatunnel-examples/pom.xml
new file mode 100644
index 0000000..4edb908
--- /dev/null
+++ b/seatunnel-examples/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>seatunnel</artifactId>
+ <groupId>org.apache.seatunnel</groupId>
+ <version>2.0.5-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>seatunnel-examples</artifactId>
+ <packaging>pom</packaging>
+ <modules>
+ <module>seatunnel-flink-examples</module>
+ </modules>
+</project>
\ No newline at end of file
diff --git a/seatunnel-examples/seatunnel-flink-examples/pom.xml b/seatunnel-examples/seatunnel-flink-examples/pom.xml
new file mode 100644
index 0000000..dfe3448
--- /dev/null
+++ b/seatunnel-examples/seatunnel-flink-examples/pom.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>seatunnel-examples</artifactId>
+ <groupId>org.apache.seatunnel</groupId>
+ <version>2.0.5-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>seatunnel-flink-examples</artifactId>
+ <properties>
+ <flink.scope>compile</flink.scope>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-core-flink</artifactId>
+ <version>${parent.version}</version>
+ </dependency>
+
+ <!--flink-->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java</artifactId>
+ <version>${flink.version}</version>
+ <scope>${flink.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+ <version>${flink.version}</version>
+ <scope>${flink.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
+ <version>${flink.version}</version>
+ <scope>${flink.scope}</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
+ <version>${flink.version}</version>
+ <scope>${flink.scope}</scope>
+ </dependency>
+
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/test/flink/LocalTest.java b/seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/test/flink/LocalTest.java
new file mode 100644
index 0000000..7dd75d5
--- /dev/null
+++ b/seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/test/flink/LocalTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.test.flink;
+
+import static org.apache.seatunnel.utils.Engine.FLINK;
+
+import org.apache.seatunnel.Seatunnel;
+import org.apache.seatunnel.config.command.CommandLineArgs;
+
+public class LocalTest {
+
+ public static final String TEST_RESOURCE_DIR = "/seatunnel-examples/seatunnel-flink-examples/src/main/resources/examples/";
+
+ public static void main(String[] args) {
+ String configFile = getTestConfigFile("fake_to_console.config");
+ CommandLineArgs flinkArgs = new CommandLineArgs(configFile, false);
+ Seatunnel.run(flinkArgs, FLINK, args);
+ }
+
+ public static String getTestConfigFile(String configFile) {
+ return System.getProperty("user.dir") + TEST_RESOURCE_DIR + configFile;
+ }
+}
diff --git a/seatunnel-examples/seatunnel-flink-examples/src/main/resources/examples/fake_to_console.config b/seatunnel-examples/seatunnel-flink-examples/src/main/resources/examples/fake_to_console.config
new file mode 100644
index 0000000..95ae594
--- /dev/null
+++ b/seatunnel-examples/seatunnel-flink-examples/src/main/resources/examples/fake_to_console.config
@@ -0,0 +1,55 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+######
+###### This config file is a demonstration of streaming processing in seatunnel config
+######
+
+env {
+ # You can set flink configuration here
+ execution.parallelism = 1
+ #execution.checkpoint.interval = 10000
+ #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint"
+}
+
+source {
+ # This is a example source plugin **only for test and demonstrate the feature source plugin**
+ FakeSourceStream {
+ result_table_name = "fake"
+ field_name = "name,age"
+ }
+
+ # If you would like to get more information about how to configure seatunnel and see full list of source plugins,
+ # please go to https://interestinglab.github.io/seatunnel-docs/#/
+}
+
+transform {
+ sql {
+ sql = "select name,age from fake"
+ }
+
+ # If you would like to get more information about how to configure seatunnel and see full list of transform plugins,
+ # please go to https://interestinglab.github.io/seatunnel-docs/#/
+}
+
+sink {
+ ConsoleSink {}
+
+
+ # If you would like to get more information about how to configure seatunnel and see full list of sink plugins,
+ # please go to https://interestinglab.github.io/seatunnel-docs/#/
+}
+
diff --git a/seatunnel-examples/seatunnel-flink-examples/src/main/resources/log4j.properties b/seatunnel-examples/seatunnel-flink-examples/src/main/resources/log4j.properties
new file mode 100644
index 0000000..db5d9e5
--- /dev/null
+++ b/seatunnel-examples/seatunnel-flink-examples/src/main/resources/log4j.properties
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Set everything to be logged to the console
+log4j.rootCategory=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n