You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ki...@apache.org on 2022/07/25 01:39:12 UTC
[incubator-seatunnel] branch 2.1.3-prepare updated (b65a2d4c9 -> 54eb70d49)
This is an automated email from the ASF dual-hosted git repository.
kirs pushed a change to branch 2.1.3-prepare
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
from b65a2d4c9 [1.3.7-prepare] allow merge button
new 1800893cd [Bug][build] Fix build failed (#1980)
new e58bc54d6 [Improvement][connector-spark-tidb] Refactored config parameters (#1983)
new 7025f3186 [Spark] Use higher version of the libthrift dependency (#1994)
new 768dc195d docs error (#1996)
new 6fae965c5 [Bug][seatunnel-core-flink] Updated FlinkRunMode enum to get the proper help message for run modes. (#2008)
new cb85bb584 a more understandable code,and code warning will disappear (#2005)
new b273ecfe3 fix same source and sink registerplugin librarycache errpr (#2015)
new fde0098d7 [Doc][Transform] Add uuid, udf, replace transform to doc (#2016)
new f5c382104 [Feature][Connector]add AssertSink connector (#2022)
new ea47bf033 Add include upper bound (<=) (#2033)
new 4c7f69b31 [Jackson] [Core] fix jackson type convert error (#2031)
new 8271709c7 [Doc] update seatunnel doc on kubernetes (#2035)
new 6c732d745 [Feature][Transform] data quality for null data rate (#1978)
new 9e94430bb [Feature][Doc] Add Flink SQL module to website. #2021 (#2056)
new 553e46a95 fix VariablesSubstituteTest create time now twice (#2063)
new 0aa076f31 Support Rsync to transfer clickhouse data file (#2074)
new 01aca5133 [Feature][Connector] add IT for Assert Sink in e2e module (#2036)
new dcba192dc [Bug] [seatunnel-connector-flink-fake] Type is not supported: BigInteger (#2118)
new 922676969 Fix the bug Unrecognized field "TwoPhaseCommit" after doris 0.15 (#2054)
new 5f86cd980 Fix the data output exception when accessing Hive using Spark JDBC So… (#2085)
new 2dbe757ff Update license.yml (#2153)
new b48ccaf0a [Bug] [seatunnel-core] Failed to get APP_DIR path bug fixed (#2165)
new 533ceae35 fix KafkaTableStream source json parse issue (#2168)
new 8c54a07b2 [Feature][transform] Add a module to set default value for null field (#1958)
new f0d0dfd71 [Bug][Connector]Hudi Source loads the data twice
new 4e23f2c22 [Bug] [seatunnel-api-flink] Connectors dependencies repeat additions (#2207)
new 2d0c4b681 [Docs]Fix Flink engine version requirements (#2220)
new fe2c30a51 [Bug] [seatunnel-core] Failed to get APP_DIR path bug fixed (#2165)
new bf48cc417 update merge button
new 54eb70d49 Merge pull request #2239 from CalvinKirs/2.1.3-prepare
The 1742 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/license.yml | 2 +-
.gitignore | 1 +
docs/en/connector/flink-sql/usage.md | 275 +++++++++++++++++++++
docs/en/connector/sink/Assert.md | 105 ++++++++
docs/en/connector/source/Fake.mdx | 2 +-
docs/en/connector/source/Socket.mdx | 8 +-
docs/en/contribution/contribute-plugin.md | 2 +-
docs/en/start/kubernetes.mdx | 38 +--
docs/en/start/local.mdx | 2 +-
docs/en/transform/nullRate.md | 67 +++++
docs/en/transform/nulltf.md | 73 ++++++
docs/sidebars.js | 21 ++
pom.xml | 37 ++-
.../apache/seatunnel/apis/base/env/RuntimeEnv.java | 4 +-
.../apache/seatunnel/flink/FlinkEnvironment.java | 7 +-
.../seatunnel/flink/batch/FlinkBatchExecution.java | 2 +-
.../apache/seatunnel/flink/util/SchemaUtil.java | 57 +++--
.../apache/seatunnel/spark/SparkEnvironment.java | 4 +-
.../apache/seatunnel/common/utils/JsonUtils.java | 5 +
.../common/utils/VariablesSubstituteTest.java | 7 +-
seatunnel-config/pom.xml | 2 +-
seatunnel-config/seatunnel-config-base/pom.xml | 2 +-
seatunnel-config/seatunnel-config-shade/pom.xml | 2 +-
seatunnel-connectors/plugin-mapping.properties | 1 +
.../seatunnel-connectors-flink-dist/pom.xml | 6 +
.../seatunnel-connectors-flink/pom.xml | 1 +
.../pom.xml | 7 +-
.../seatunnel/flink/assertion/AssertExecutor.java | 105 ++++++++
.../flink/assertion/rule/AssertFieldRule.java | 64 +++++
.../flink/assertion/rule/AssertRuleParser.java | 92 +++++++
.../seatunnel/flink/assertion/sink/AssertSink.java | 117 +++++++++
.../flink/assertion/AssertExecutorTest.java | 108 ++++++++
.../flink/assertion/rule/AssertRuleParserTest.java | 83 +++++++
.../seatunnel/flink/doris/sink/RespContent.java | 12 +
.../flink/fake/source/FakeSourceStream.java | 2 +-
.../seatunnel/flink/fake/source/MockSchema.java | 19 +-
.../flink/file/source/JsonRowInputFormat.java | 27 +-
.../seatunnel/fink/file/source/FileSourceTest.java | 2 +
.../seatunnel/flink/jdbc/source/JdbcSource.java | 2 +-
.../flink/kafka/source/KafkaTableStream.java | 2 +-
.../seatunnel-connectors-spark-dist/pom.xml | 6 +
.../spark/clickhouse/sink/ClickhouseFile.scala | 31 +--
.../sink/filetransfer/RsyncFileTransfer.scala | 65 ++++-
.../apache/seatunnel/spark/hudi/source/Hudi.scala | 2 +-
.../apache/seatunnel/spark/jdbc/source/Jdbc.scala | 7 +-
.../spark/jdbc/source/util/HiveDialet.scala} | 23 +-
.../org/apache/seatunnel/spark/tidb}/Config.scala | 34 ++-
.../apache/seatunnel/spark/tidb/sink/Tidb.scala | 15 +-
.../apache/seatunnel/spark/tidb/source/Tidb.scala | 7 +-
.../seatunnel/core/base/config/PluginFactory.java | 14 +-
.../src/main/bin/start-seatunnel-sql.sh | 22 +-
.../src/main/bin/start-seatunnel-flink.sh | 25 +-
.../seatunnel/core/flink/config/FlinkRunMode.java | 7 +
seatunnel-core/seatunnel-core-spark/pom.xml | 12 +
.../src/main/bin/start-seatunnel-spark.sh | 22 +-
.../FakeSourceToAssertIT.java} | 8 +-
.../fakesource_to_assert.conf} | 42 +++-
.../seatunnel-flink-examples/pom.xml | 5 +
.../seatunnel-transforms-spark/pom.xml | 2 +
.../pom.xml | 6 +-
.../org.apache.seatunnel.spark.BaseSparkTransform | 2 +-
.../seatunnel/spark/transform/NullRate.scala | 95 +++++++
.../seatunnel/spark/transform/NullRateConfig.scala | 14 +-
.../pom.xml | 7 +-
.../org.apache.seatunnel.spark.BaseSparkTransform | 2 +-
.../apache/seatunnel/spark/transform/Nulltf.scala | 111 +++++++++
.../seatunnel/spark/transform/NulltfConfig.scala | 9 +-
67 files changed, 1789 insertions(+), 181 deletions(-)
create mode 100644 docs/en/connector/flink-sql/usage.md
create mode 100644 docs/en/connector/sink/Assert.md
create mode 100644 docs/en/transform/nullRate.md
create mode 100644 docs/en/transform/nulltf.md
copy seatunnel-connectors/seatunnel-connectors-flink/{seatunnel-connector-flink-console => seatunnel-connector-flink-assert}/pom.xml (86%)
create mode 100644 seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-assert/src/main/java/org/apache/seatunnel/flink/assertion/AssertExecutor.java
create mode 100644 seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-assert/src/main/java/org/apache/seatunnel/flink/assertion/rule/AssertFieldRule.java
create mode 100644 seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-assert/src/main/java/org/apache/seatunnel/flink/assertion/rule/AssertRuleParser.java
create mode 100644 seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-assert/src/main/java/org/apache/seatunnel/flink/assertion/sink/AssertSink.java
create mode 100644 seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-assert/src/test/java/org/apache/seatunnel/flink/assertion/AssertExecutorTest.java
create mode 100644 seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-assert/src/test/java/org/apache/seatunnel/flink/assertion/rule/AssertRuleParserTest.java
copy seatunnel-connectors/seatunnel-connectors-spark/{seatunnel-connector-spark-clickhouse/src/main/scala/org/apache/seatunnel/spark/clickhouse/sink/filetransfer/TransferMethod.scala => seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/jdbc/source/util/HiveDialet.scala} (66%)
copy seatunnel-connectors/seatunnel-connectors-spark/{seatunnel-connector-spark-iceberg/src/main/scala/org/apache/seatunnel/spark/iceberg => seatunnel-connector-spark-tidb/src/main/scala/org/apache/seatunnel/spark/tidb}/Config.scala (66%)
copy seatunnel-e2e/seatunnel-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/{fake/FakeSourceToConsoleIT.java => assertion/FakeSourceToAssertIT.java} (82%)
copy seatunnel-e2e/seatunnel-flink-e2e/src/test/resources/{file/fakesource_to_file.conf => assertion/fakesource_to_assert.conf} (66%)
copy seatunnel-transforms/seatunnel-transforms-spark/{seatunnel-transform-spark-uuid => seatunnel-transform-spark-null-rate}/pom.xml (86%)
copy seatunnel-transforms/seatunnel-transforms-spark/{seatunnel-transform-spark-json => seatunnel-transform-spark-null-rate}/src/main/resources/META-INF/services/org.apache.seatunnel.spark.BaseSparkTransform (94%)
create mode 100644 seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-null-rate/src/main/scala/org/apache/seatunnel/spark/transform/NullRate.scala
copy seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/command/CommandBuilder.java => seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-null-rate/src/main/scala/org/apache/seatunnel/spark/transform/NullRateConfig.scala (79%)
copy seatunnel-transforms/seatunnel-transforms-spark/{seatunnel-transform-spark-json => seatunnel-transform-spark-nulltf}/pom.xml (85%)
copy seatunnel-transforms/seatunnel-transforms-spark/{seatunnel-transform-spark-json => seatunnel-transform-spark-nulltf}/src/main/resources/META-INF/services/org.apache.seatunnel.spark.BaseSparkTransform (94%)
create mode 100644 seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-nulltf/src/main/scala/org/apache/seatunnel/spark/transform/Nulltf.scala
copy seatunnel-common/src/main/java/org/apache/seatunnel/common/constants/JobMode.java => seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-nulltf/src/main/scala/org/apache/seatunnel/spark/transform/NulltfConfig.scala (86%)