You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampark.apache.org by be...@apache.org on 2023/04/14 07:17:25 UTC
[incubator-streampark] branch dev updated: [Improve] spotless plugin support scala code format (#2625)
This is an automated email from the ASF dual-hosted git repository.
benjobs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampark.git
The following commit(s) were added to refs/heads/dev by this push:
new dec2af903 [Improve] spotless plugin support scala code format (#2625)
dec2af903 is described below
commit dec2af903f9381e4875bd885edc8e9d5d7720c9b
Author: zhoulii <zh...@foxmail.com>
AuthorDate: Fri Apr 14 15:17:19 2023 +0800
[Improve] spotless plugin support scala code format (#2625)
* [Improve] spotless plugin support scala code format
---------
Co-authored-by: zhoulii <zh...@hikvision.com.cn>
---
.scalafmt.conf | 87 ++++
pom.xml | 49 +-
.../streampark/common/conf/CommonConfig.scala | 12 +-
.../streampark/common/conf/ConfigConst.scala | 49 +-
.../streampark/common/conf/ConfigOption.scala | 30 +-
.../streampark/common/conf/FlinkVersion.scala | 24 +-
.../common/conf/InternalConfigHolder.scala | 90 ++--
.../streampark/common/conf/InternalOption.scala | 18 +-
.../streampark/common/conf/K8sFlinkConfig.scala | 23 +-
.../apache/streampark/common/conf/Workspace.scala | 37 +-
.../streampark/common/enums/RestartStrategy.scala | 4 +-
.../apache/streampark/common/fs/FsOperator.scala | 37 +-
.../apache/streampark/common/fs/HdfsOperator.scala | 29 +-
.../apache/streampark/common/fs/LfsOperator.scala | 63 +--
.../streampark/common/util/ClassLoaderUtils.scala | 20 +-
.../streampark/common/util/CommandUtils.scala | 12 +-
.../common/util/CompletableFutureUtils.scala | 72 +--
.../streampark/common/util/ConfigUtils.scala | 46 +-
.../apache/streampark/common/util/DateUtils.scala | 46 +-
.../streampark/common/util/DeflaterUtils.scala | 4 +-
.../apache/streampark/common/util/FileUtils.scala | 28 +-
.../streampark/common/util/HBaseClient.scala | 34 +-
.../streampark/common/util/HadoopConfigUtils.scala | 85 ++--
.../streampark/common/util/HadoopUtils.scala | 98 ++--
.../apache/streampark/common/util/HdfsUtils.scala | 69 ++-
.../apache/streampark/common/util/HostsUtils.scala | 24 +-
.../streampark/common/util/HttpClientUtils.scala | 84 ++--
.../apache/streampark/common/util/JdbcUtils.scala | 159 ++++---
.../apache/streampark/common/util/JsonUtils.scala | 11 +-
.../org/apache/streampark/common/util/Logger.scala | 48 +-
.../streampark/common/util/MongoConfig.scala | 45 +-
.../streampark/common/util/PropertiesUtils.scala | 137 ++++--
.../streampark/common/util/RedisClient.scala | 27 +-
.../streampark/common/util/RedisEndpoint.scala | 207 +++++----
.../apache/streampark/common/util/RedisUtils.scala | 167 ++++---
.../streampark/common/util/ReflectUtils.scala | 29 +-
.../streampark/common/util/SqlConvertUtils.scala | 70 +--
.../common/util/SystemPropertyUtils.scala | 32 +-
.../streampark/common/util/ThreadUtils.scala | 4 +-
.../org/apache/streampark/common/util/Utils.scala | 65 ++-
.../apache/streampark/common/util/YarnUtils.scala | 175 +++----
.../streampark/common/util/ZooKeeperUtils.scala | 35 +-
.../streampark/common/fs/LfsOperatorTest.scala | 451 +++++++++---------
.../org/apache/streampark/common/fs/package.scala | 20 +-
.../streampark/flink/client/FlinkClient.scala | 47 +-
.../flink/client/bean/CancelRequest.scala | 29 +-
.../flink/client/bean/DeployRequest.scala | 43 +-
.../flink/client/bean/SavepointRequestTrait.scala | 4 +-
.../flink/client/bean/ShutDownRequest.scala | 18 +-
.../flink/client/bean/SubmitRequest.scala | 134 +++---
.../flink/client/bean/SubmitResponse.scala | 9 +-
.../client/bean/TriggerSavepointRequest.scala | 20 +-
.../flink/client/FlinkClientHandler.scala | 18 +-
.../impl/KubernetesNativeApplicationClient.scala | 48 +-
.../impl/KubernetesNativeSessionClient.scala | 152 +++---
.../streampark/flink/client/impl/LocalClient.scala | 34 +-
.../flink/client/impl/RemoteClient.scala | 104 +++--
.../flink/client/impl/YarnApplicationClient.scala | 86 ++--
.../flink/client/impl/YarnPerJobClient.scala | 83 ++--
.../flink/client/impl/YarnSessionClient.scala | 161 ++++---
.../client/tool/FlinkSessionClientHelper.scala | 65 ++-
.../flink/client/trait/FlinkClientTrait.scala | 257 +++++++----
.../client/trait/KubernetesNativeClientTrait.scala | 111 +++--
.../flink/client/trait/YarnClientTrait.scala | 78 ++--
.../flink/client/test/ParameterTestCase.scala | 11 +-
.../flink/client/test/YarnPerJobTestCase.scala | 110 ++---
.../flink/connector/conf/ThresholdConf.scala | 10 +-
.../connector/conf/ThresholdConfigOption.scala | 10 +-
.../flink/connector/failover/FailoverChecker.scala | 13 +-
.../flink/connector/failover/FailoverWriter.scala | 63 ++-
.../flink/connector/failover/SinkBuffer.scala | 11 +-
.../flink/connector/failover/SinkRequest.scala | 29 +-
.../streampark/flink/connector/sink/Sink.scala | 6 +-
.../clickhouse/conf/ClickHouseHttpConfig.scala | 12 +-
.../conf/ClickHouseSinkConfigOption.scala | 49 +-
.../internal/AsyncClickHouseSinkFunction.scala | 24 +-
.../internal/ClickHouseSinkFunction.scala | 71 +--
.../clickhouse/internal/ClickHouseSinkWriter.scala | 34 +-
.../clickhouse/internal/ClickHouseWriterTask.scala | 56 ++-
.../sink/AsyncClickHouseOutputFormat.scala | 12 +-
.../clickhouse/sink/ClickHouseOutputFormat.scala | 12 +-
.../connector/clickhouse/sink/ClickHouseSink.scala | 49 +-
.../clickhouse/util/ClickhouseConvertUtils.scala | 17 +-
.../connector/doris/conf/DorisConfig.scala | 4 +-
.../doris/conf/DorisSinkConfigOption.scala | 53 +--
.../elasticsearch5/conf/ESSinkConfigOption.scala | 26 +-
.../elasticsearch5/internal/ESSinkFunction.scala | 31 +-
.../connector/elasticsearch5/sink/ES5Sink.scala | 41 +-
.../elasticsearch5/util/ElasticsearchUtils.scala | 12 +-
.../bean/RestClientFactoryImpl.scala | 19 +-
.../connector/elasticsearch6/conf/ES6Config.scala | 4 +-
.../elasticsearch6/conf/ES6SinkConfigOption.scala | 30 +-
.../elasticsearch6/internal/ESSinkFunction.scala | 31 +-
.../connector/elasticsearch6/sink/ES6Sink.scala | 56 ++-
.../elasticsearch6/util/ElasticsearchUtils.scala | 12 +-
.../bean/RestClientFactoryImpl.scala | 19 +-
.../connector/elasticsearch7/conf/ES7Config.scala | 4 +-
.../elasticsearch7/conf/ES7SinkConfigOption.scala | 30 +-
.../elasticsearch7/internal/ESSinkFunction.scala | 17 +-
.../connector/elasticsearch7/sink/ES7Sink.scala | 58 ++-
.../elasticsearch7/util/ElasticsearchUtils.scala | 12 +-
.../hbase/internal/HBaseSinkFunction.scala | 50 +-
.../hbase/internal/HBaseSourceFunction.scala | 83 ++--
.../connector/hbase/request/HBaseRequest.scala | 88 ++--
.../connector/hbase/sink/HBaseOutputFormat.scala | 19 +-
.../flink/connector/hbase/sink/HBaseSink.scala | 36 +-
.../flink/connector/hbase/source/HBaseSource.scala | 29 +-
.../connector/http/conf/HttpConfigOption.scala | 5 +-
.../connector/http/function/HttpSinkFunction.scala | 27 +-
.../connector/http/internal/HttpSinkWriter.scala | 28 +-
.../connector/http/internal/HttpWriterTask.scala | 70 +--
.../flink/connector/http/sink/HttpSink.scala | 55 ++-
.../flink/connector/influx/bean/InfluxEntity.scala | 35 +-
.../connector/influx/function/InfluxFunction.scala | 23 +-
.../connector/influx/sink/InfluxOutputFormat.scala | 12 +-
.../flink/connector/influx/sink/InfluxSink.scala | 23 +-
.../flink/connector/jdbc/bean/Transaction.scala | 10 +-
.../jdbc/internal/Jdbc2PCSinkFunction.scala | 38 +-
.../jdbc/internal/JdbcASyncFunction.scala | 45 +-
.../connector/jdbc/internal/JdbcSinkFunction.scala | 22 +-
.../jdbc/internal/JdbcSourceFunction.scala | 75 +--
.../flink/connector/jdbc/request/JdbcRequest.scala | 20 +-
.../flink/connector/jdbc/sink/JdbcSink.scala | 54 ++-
.../flink/connector/jdbc/source/JdbcSource.scala | 27 +-
.../kafka/bean/KafkaEqualityPartitioner.scala | 27 +-
.../flink/connector/kafka/sink/KafkaSink.scala | 67 +--
.../flink/connector/kafka/source/KafkaSource.scala | 154 ++++---
.../mongo/internal/MongoSourceFunction.scala | 69 +--
.../flink/connector/mongo/source/MongoSource.scala | 19 +-
.../connector/redis/bean/RedisContainer.scala | 40 +-
.../flink/connector/redis/bean/RedisMapper.scala | 34 +-
.../connector/redis/bean/RedisTransaction.scala | 13 +-
.../flink/connector/redis/conf/RedisConfig.scala | 24 +-
.../redis/conf/RedisSinkConfigOption.scala | 22 +-
.../redis/internal/Redis2PCSinkFunction.scala | 29 +-
.../redis/internal/RedisSinkFunction.scala | 10 +-
.../flink/connector/redis/sink/RedisSink.scala | 129 +++---
.../flink/core/scala/DataStreamExt.scala | 70 ++-
.../flink/core/scala/FlinkStreamTable.scala | 16 +-
.../flink/core/scala/FlinkStreaming.scala | 35 +-
.../streampark/flink/core/scala/FlinkTable.scala | 8 +-
.../flink/kubernetes/ChangeEventBus.scala | 7 +-
.../flink/kubernetes/DefaultFlinkK8sWatcher.scala | 47 +-
.../flink/kubernetes/FlinkK8sWatchController.scala | 58 ++-
.../flink/kubernetes/FlinkK8sWatcher.scala | 92 ++--
.../kubernetes/FlinkK8sWatcherLazyStartAop.scala | 5 +-
.../flink/kubernetes/IngressController.scala | 100 ++--
.../flink/kubernetes/KubernetesRetriever.scala | 81 ++--
.../flink/kubernetes/PodTemplateParser.scala | 75 +--
.../flink/kubernetes/PodTemplateTool.scala | 39 +-
.../streampark/flink/kubernetes/TrackConfig.scala | 60 +--
.../flink/kubernetes/enums/FlinkJobState.scala | 7 +-
.../kubernetes/enums/FlinkK8sExecuteMode.scala | 4 +-
.../event/FlinkClusterMetricChangeEvent.scala | 3 +-
.../event/FlinkJobCheckpointChangeEvent.scala | 7 +-
.../kubernetes/event/FlinkJobStateEvent.scala | 6 +-
.../event/FlinkJobStatusChangeEvent.scala | 4 +-
.../helper/KubernetesDeploymentHelper.scala | 117 +++--
.../helper/KubernetesWatcherHelper.scala | 44 +-
.../flink/kubernetes/model/CheckpointCV.scala | 8 +-
.../flink/kubernetes/model/ClusterKey.scala | 12 +-
.../flink/kubernetes/model/FlinkMetricCV.scala | 53 +--
.../flink/kubernetes/model/JobStatusCV.scala | 18 +-
.../kubernetes/model/K8sDeploymentEventCV.scala | 9 +-
.../flink/kubernetes/model/K8sPodTemplates.scala | 13 +-
.../flink/kubernetes/model/TrackId.scala | 42 +-
.../watcher/FlinkCheckpointWatcher.scala | 113 +++--
.../kubernetes/watcher/FlinkJobStatusWatcher.scala | 416 +++++++++--------
.../kubernetes/watcher/FlinkK8sEventWatcher.scala | 33 +-
.../kubernetes/watcher/FlinkMetricsWatcher.scala | 167 +++----
.../flink/kubernetes/watcher/FlinkWatcher.scala | 18 +-
.../flink/kubernetes/FlinkRestJsonTest.scala | 81 ++--
.../flink/kubernetes/PodTemplateParserTest.scala | 22 +-
.../streampark/flink/packer/PackerResourceGC.scala | 20 +-
.../flink/packer/docker/DockerConf.scala | 31 +-
.../flink/packer/docker/DockerRetriever.scala | 39 +-
.../packer/docker/FlinkDockerfileTemplate.scala | 24 +-
.../docker/FlinkDockerfileTemplateTrait.scala | 65 ++-
.../docker/FlinkHadoopDockerfileTemplate.scala | 76 +--
.../streampark/flink/packer/docker/package.scala | 4 +-
.../streampark/flink/packer/maven/Artifact.scala | 14 +-
.../flink/packer/maven/DependencyInfo.scala | 6 +-
.../streampark/flink/packer/maven/MavenTool.scala | 137 +++---
.../flink/packer/pipeline/BuildPipeline.scala | 93 ++--
.../flink/packer/pipeline/BuildRequest.scala | 25 +-
.../flink/packer/pipeline/BuildResponse.scala | 29 +-
.../packer/pipeline/DockerProgressWatcher.scala | 20 +-
.../packer/pipeline/DockerResolveProgress.scala | 46 +-
.../packer/pipeline/DockerResolveSnapshot.scala | 53 ++-
.../flink/packer/pipeline/PipeError.scala | 20 +-
.../flink/packer/pipeline/PipeSnapshot.scala | 19 +-
.../flink/packer/pipeline/PipeWatcher.scala | 16 +-
.../impl/FlinkK8sApplicationBuildPipeline.scala | 88 ++--
.../impl/FlinkK8sSessionBuildPipeline.scala | 16 +-
.../pipeline/impl/FlinkRemoteBuildPipeline.scala | 16 +-
.../impl/FlinkYarnApplicationBuildPipeline.scala | 47 +-
.../streampark/flink/packer/DockerClientTest.scala | 21 +-
.../flink/packer/FlinkDockerfileTemplateSpec.scala | 10 +-
.../flink/packer/MavenArtifactSpec.scala | 17 +-
.../streampark/flink/packer/MavenToolSpec.scala | 66 ++-
.../apache/streampark/flink/packer/package.scala | 4 +-
.../flink/proxy/ChildFirstClassLoader.scala | 31 +-
.../flink/proxy/ClassLoaderObjectInputStream.scala | 44 +-
.../streampark/flink/proxy/FlinkShimsProxy.scala | 125 ++---
.../streampark/flink/core/EnhancerImplicit.scala | 6 +-
.../streampark/flink/core/FlinkClientTrait.scala | 7 +-
.../flink/core/FlinkKubernetesClientTrait.scala | 9 +-
.../streampark/flink/core/FlinkSqlExecutor.scala | 181 ++++----
.../streampark/flink/core/FlinkSqlValidator.scala | 42 +-
.../flink/core/FlinkStreamTableTrait.scala | 250 ++++++----
.../flink/core/FlinkStreamingInitializer.scala | 51 +-
.../flink/core/FlinkTableInitializer.scala | 93 ++--
.../streampark/flink/core/FlinkTableTrait.scala | 67 ++-
.../streampark/flink/core/SqlCommandParser.scala | 512 ++++++++-------------
.../streampark/flink/core/StreamEnvConfig.scala | 5 +-
.../flink/core/conf/FlinkConfiguration.scala | 5 +-
.../flink/core/conf/FlinkRunOption.scala | 83 +++-
.../streampark/flink/core/conf/ParameterCli.scala | 108 +++--
.../apache/streampark/flink/util/FlinkUtils.scala | 23 +-
.../flink/core/test/FlinkSqlExecuteFunSuite.scala | 93 ++--
.../core/test/FlinkSqlValidationFunSuite.scala | 269 +++++------
.../streampark/flink/core/FlinkClusterClient.scala | 3 +-
.../flink/core/FlinkKubernetesClient.scala | 3 +-
.../streampark/flink/core/StreamTableContext.scala | 48 +-
.../streampark/flink/core/TableContext.scala | 25 +-
.../apache/streampark/flink/core/TableExt.scala | 8 +-
.../streampark/flink/core/FlinkClusterClient.scala | 3 +-
.../flink/core/FlinkKubernetesClient.scala | 3 +-
.../streampark/flink/core/StreamTableContext.scala | 67 ++-
.../streampark/flink/core/TableContext.scala | 25 +-
.../apache/streampark/flink/core/TableExt.scala | 8 +-
.../streampark/flink/core/FlinkClusterClient.scala | 3 +-
.../flink/core/FlinkKubernetesClient.scala | 3 +-
.../streampark/flink/core/StreamTableContext.scala | 66 ++-
.../streampark/flink/core/TableContext.scala | 21 +-
.../apache/streampark/flink/core/TableExt.scala | 8 +-
.../streampark/flink/core/FlinkClusterClient.scala | 22 +-
.../flink/core/FlinkKubernetesClient.scala | 3 +-
.../streampark/flink/core/StreamTableContext.scala | 55 ++-
.../streampark/flink/core/TableContext.scala | 25 +-
.../streampark/flink/core/FlinkClusterClient.scala | 22 +-
.../flink/core/FlinkKubernetesClient.scala | 3 +-
.../streampark/flink/core/StreamTableContext.scala | 55 ++-
.../streampark/flink/core/TableContext.scala | 20 +-
.../streampark/flink/core/FlinkClusterClient.scala | 18 +-
.../flink/core/FlinkKubernetesClient.scala | 3 +-
.../streampark/flink/core/StreamTableContext.scala | 114 +++--
.../streampark/flink/core/TableContext.scala | 80 ++--
.../apache/streampark/flink/cli/SqlClient.scala | 19 +-
.../streampark/spark/connector/sink/ShowSink.scala | 5 +-
.../streampark/spark/connector/sink/Sink.scala | 31 +-
.../streampark/spark/connector/source/Source.scala | 16 +-
.../connector/kafka/offset/DefaultOffset.scala | 4 +-
.../spark/connector/kafka/offset/HBaseOffset.scala | 103 +++--
.../spark/connector/kafka/offset/KafkaClient.scala | 25 +-
.../spark/connector/kafka/offset/MySQLOffset.scala | 97 ++--
.../spark/connector/kafka/offset/Offset.scala | 47 +-
.../spark/connector/kafka/offset/RedisOffset.scala | 51 +-
.../spark/connector/kafka/sink/KafkaSink.scala | 24 +-
.../spark/connector/kafka/source/KafkaSource.scala | 62 +--
.../kafka/writer/DStreamKafkaWriter.scala | 33 +-
.../connector/kafka/writer/IterKafkaWriter.scala | 25 +-
.../spark/connector/kafka/writer/KafkaWriter.scala | 67 +--
.../connector/kafka/writer/RDDKafkaWriter.scala | 32 +-
.../connector/kafka/writer/SimpleKafkaWriter.scala | 25 +-
.../org/apache/streampark/spark/core/Spark.scala | 89 ++--
.../apache/streampark/spark/core/SparkBatch.scala | 11 +-
.../streampark/spark/core/SparkStreaming.scala | 31 +-
.../serializable/MultipleAvroOutputsFormat.scala | 11 +-
.../spark/core/serializable/MultipleOutputer.scala | 12 +-
.../core/serializable/MultipleOutputsFormat.scala | 66 +--
.../serializable/MultipleTextOutputsFormat.scala | 2 -
.../serializable/SerializationExtensions.scala | 41 +-
.../apache/streampark/spark/core/sink/EsSink.scala | 51 --
.../streampark/spark/core/sink/HBaseSink.scala | 96 ----
.../streampark/spark/core/sink/MySQLSink.scala | 79 ----
.../streampark/spark/core/sink/RedisSink.scala | 77 ----
.../streampark/spark/core/sink/RichHBSink.scala | 92 ----
.../spark/core/util/SQLContextUtil.scala | 15 +-
tools/checkstyle/scalastyle-config.xml | 301 ------------
280 files changed, 7711 insertions(+), 6485 deletions(-)
diff --git a/.scalafmt.conf b/.scalafmt.conf
new file mode 100644
index 000000000..eac72aacd
--- /dev/null
+++ b/.scalafmt.conf
@@ -0,0 +1,87 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+runner.dialect = scala212
+
+# Version is required to make sure IntelliJ picks the right version
+version = 3.4.3
+preset = default
+
+# Max column
+maxColumn = 100
+
+# This parameter simply says the .stripMargin method was not redefined by the user to assign
+# special meaning to indentation preceding the | character. Hence, that indentation can be modified.
+assumeStandardLibraryStripMargin = true
+align.stripMargin = true
+
+# Align settings
+align.preset = none
+align.closeParenSite = false
+align.openParenCallSite = false
+danglingParentheses.defnSite = false
+danglingParentheses.callSite = false
+danglingParentheses.ctrlSite = true
+danglingParentheses.tupleSite = false
+align.openParenCallSite = false
+align.openParenDefnSite = false
+align.openParenTupleSite = false
+
+# Newlines
+newlines.alwaysBeforeElseAfterCurlyIf = false
+newlines.beforeCurlyLambdaParams = multiline # Newline before lambda params
+newlines.afterCurlyLambdaParams = squash # No newline after lambda params
+newlines.inInterpolation = "avoid"
+newlines.avoidInResultType = true
+optIn.annotationNewlines = true
+
+# Scaladoc
+docstrings.style = Asterisk # Javadoc style
+docstrings.removeEmpty = true
+docstrings.oneline = fold
+docstrings.forceBlankLineBefore = true
+
+# Indentation
+indent.extendSite = 2 # This makes sure extend is not indented as the ctor parameters
+
+# Rewrites
+rewrite.rules = [AvoidInfix, Imports, RedundantBraces, SortModifiers]
+
+# Imports
+rewrite.imports.sort = scalastyle
+rewrite.imports.groups = [
+ ["org.apache.streampark\\..*"],
+ ["org.apache.streampark.shaded\\..*"],
+ [".*"],
+ ["javax\\..*"],
+ ["java\\..*"],
+ ["scala\\..*"]
+]
+rewrite.imports.contiguousGroups = no
+importSelectors = singleline # Imports in a single line, like IntelliJ
+
+# Remove redundant braces in string interpolation.
+rewrite.redundantBraces.stringInterpolation = true
+rewrite.redundantBraces.defnBodies = false
+rewrite.redundantBraces.generalExpressions = false
+rewrite.redundantBraces.ifElseExpressions = false
+rewrite.redundantBraces.methodBodies = false
+rewrite.redundantBraces.includeUnitMethods = false
+rewrite.redundantBraces.maxBreaks = 1
+
+# Remove trailing commas
+rewrite.trailingCommas.style = "never"
diff --git a/pom.xml b/pom.xml
index 57f1e307d..40801c4c4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -130,10 +130,9 @@
<maven-javadoc-plugin.version>2.9.1</maven-javadoc-plugin.version>
<maven-source-plugin.version>2.2.1</maven-source-plugin.version>
<maven-spotless-plugin.version>2.27.2</maven-spotless-plugin.version>
- <maven-scalastyle-plugin.version>1.0.0</maven-scalastyle-plugin.version>
+ <spotless.scalafmt.version>3.4.3</spotless.scalafmt.version>
<maven-checkstyle-plugin.version>3.2.0</maven-checkstyle-plugin.version>
<build-helper-maven-plugin.version>3.3.0</build-helper-maven-plugin.version>
- <scalastyle.fails.on.error>true</scalastyle.fails.on.error>
<streampark.shaded.package>org.apache.streampark.shaded</streampark.shaded.package>
<flink.table.uber.artifact.id>flink-table-uber_${scala.binary.version}</flink.table.uber.artifact.id>
<httpclient.version>4.5.13</httpclient.version>
@@ -647,41 +646,6 @@
</executions>
</plugin>
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <version>${maven-scalastyle-plugin.version}</version>
- <configuration>
- <verbose>false</verbose>
- <failOnViolation>${scalastyle.fails.on.error}</failOnViolation>
- <includeTestSourceDirectory>true</includeTestSourceDirectory>
- <failOnWarning>false</failOnWarning>
- <sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
- <testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
- <!--suppress UnresolvedMavenProperty -->
- <configLocation>${maven.multiModuleProjectDirectory}/tools/checkstyle/scalastyle-config.xml</configLocation>
- <outputFile>${project.build.directory}/target/scalastyle-output.xml</outputFile>
- <inputEncoding>UTF-8</inputEncoding>
- <outputEncoding>UTF-8</outputEncoding>
- </configuration>
- <dependencies>
- <dependency>
- <groupId>com.puppycrawl.tools</groupId>
- <artifactId>checkstyle</artifactId>
- <version>8.14</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <id>validate</id>
- <goals>
- <goal>check</goal>
- </goals>
- <phase>process-sources</phase>
- </execution>
- </executions>
- </plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
@@ -766,6 +730,12 @@
</replaceRegex>
<removeUnusedImports />
</java>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>${maven.multiModuleProjectDirectory}/.scalafmt.conf</file>
+ </scalafmt>
+ </scala>
</configuration>
<executions>
<execution>
@@ -939,11 +909,6 @@
<groupId>com.diffplug.spotless</groupId>
<artifactId>spotless-maven-plugin</artifactId>
</plugin>
-
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- </plugin>
</plugins>
</build>
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/conf/CommonConfig.scala b/streampark-common/src/main/scala/org/apache/streampark/common/conf/CommonConfig.scala
index 1e6256120..18d53a35f 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/conf/CommonConfig.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/conf/CommonConfig.scala
@@ -57,19 +57,22 @@ object CommonConfig {
key = "streampark.docker.http-client.max-connections",
defaultValue = 100,
classType = classOf[JavaInt],
- description = "instantiating max connections for DockerHttpClient")
+ description = "instantiating max connections for DockerHttpClient"
+ )
val DOCKER_CONNECTION_TIMEOUT_SEC: InternalOption = InternalOption(
key = "streampark.docker.http-client.connection-timeout-sec",
defaultValue = 100L,
classType = classOf[JavaLong],
- description = "instantiating connection timeout for DockerHttpClient")
+ description = "instantiating connection timeout for DockerHttpClient"
+ )
val DOCKER_RESPONSE_TIMEOUT_SEC: InternalOption = InternalOption(
key = "streampark.docker.http-client.response-timeout-sec",
defaultValue = 120L,
classType = classOf[JavaLong],
- description = "instantiating connection timeout for DockerHttpClient")
+ description = "instantiating connection timeout for DockerHttpClient"
+ )
val MAVEN_SETTINGS_PATH: InternalOption = InternalOption(
key = "streampark.maven.settings",
@@ -81,7 +84,8 @@ object CommonConfig {
key = "streampark.maven.central.repository",
defaultValue = "https://repo1.maven.org/maven2/",
classType = classOf[String],
- description = "maven repository used for built-in compilation")
+ description = "maven repository used for built-in compilation"
+ )
val MAVEN_AUTH_USER: InternalOption = InternalOption(
key = "streampark.maven.auth.user",
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/conf/ConfigConst.scala b/streampark-common/src/main/scala/org/apache/streampark/common/conf/ConfigConst.scala
index d31049166..ccbe4e8ff 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/conf/ConfigConst.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/conf/ConfigConst.scala
@@ -20,15 +20,11 @@ import java.time.LocalDateTime
object ConfigConst {
- /**
- * common const
- */
+ /** common const */
val DEFAULT_DATAMASK_STRING = "********"
- /**
- * about parameter...
- */
+ /** about parameter... */
val KEY_APP_HOME = "app.home"
@@ -46,9 +42,7 @@ object ConfigConst {
val KEY_SEMANTIC = "semantic"
- /**
- * sign....
- */
+ /** sign.... */
val SIGN_COLON = ":"
val SIGN_SEMICOLON = ";"
@@ -57,16 +51,12 @@ object ConfigConst {
val SIGN_EMPTY = ""
- /**
- * kerberos
- */
+ /** kerberos */
val KEY_KERBEROS = "kerberos"
val KEY_HADOOP_USER_NAME = "HADOOP_USER_NAME"
- /**
- * hadoop.security.authentication
- */
+ /** hadoop.security.authentication */
val KEY_HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"
val KEY_SECURITY_KERBEROS_ENABLE = "security.kerberos.login.enable"
@@ -92,13 +82,16 @@ object ConfigConst {
// flink
def KEY_APP_CONF(prefix: String = null): String = if (prefix == null) "conf" else s"${prefix}conf"
- def KEY_FLINK_CONF(prefix: String = null): String = if (prefix == null) "flink.conf" else s"${prefix}flink.conf"
+ def KEY_FLINK_CONF(prefix: String = null): String =
+ if (prefix == null) "flink.conf" else s"${prefix}flink.conf"
- def KEY_APP_NAME(prefix: String = null): String = if (prefix == null) "app.name" else s"${prefix}app.name"
+ def KEY_APP_NAME(prefix: String = null): String =
+ if (prefix == null) "app.name" else s"${prefix}app.name"
def KEY_FLINK_SQL(prefix: String = null): String = if (prefix == null) "sql" else s"${prefix}sql"
- def KEY_FLINK_PARALLELISM(prefix: String = null): String = if (prefix == null) "parallelism.default" else s"${prefix}parallelism.default"
+ def KEY_FLINK_PARALLELISM(prefix: String = null): String =
+ if (prefix == null) "parallelism.default" else s"${prefix}parallelism.default"
val KEY_FLINK_OPTION_PREFIX = "flink.option."
@@ -131,9 +124,7 @@ object ConfigConst {
val KEY_FLINK_TABLE_DATABASE = "flink.table.database"
- /**
- * about config Kafka
- */
+ /** about config Kafka */
val KAFKA_SINK_PREFIX = "kafka.sink."
@@ -153,9 +144,7 @@ object ConfigConst {
val KEY_ALIAS = "alias"
- /**
- * about config jdbc...
- */
+ /** about config jdbc... */
val KEY_JDBC_PREFIX = "jdbc."
val KEY_JDBC_DRIVER = "driverClassName"
@@ -172,9 +161,7 @@ object ConfigConst {
val MONGO_PREFIX = "mongodb."
- /**
- * about config HBase
- */
+ /** about config HBase */
val HBASE_PREFIX = "hbase."
val KEY_HBASE_COMMIT_BATCH = "hbase.commit.batch"
@@ -187,18 +174,14 @@ object ConfigConst {
val DEFAULT_HBASE_WRITE_SIZE = 1024 * 1024 * 10
- /**
- * about influx
- */
+ /** about influx */
val INFLUX_PREFIX = "influx."
val KEY_INFLUX_ACTIONS = "actions"
val KEY_INFLUX_FLUSH_DURATION = "flush.duration"
- /**
- * flink config key
- */
+ /** flink config key */
val KEY_FLINK_APPLICATION_ARGS = "$internal.application.program-args"
val KEY_FLINK_APPLICATION_MAIN_CLASS = "$internal.application.main"
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/conf/ConfigOption.scala b/streampark-common/src/main/scala/org/apache/streampark/common/conf/ConfigOption.scala
index 5046ed76c..500fcf3f6 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/conf/ConfigOption.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/conf/ConfigOption.scala
@@ -19,20 +19,30 @@ package org.apache.streampark.common.conf
import org.apache.streampark.common.util.Utils.StringCasts
import java.util.Properties
+
import scala.util.{Failure, Success, Try}
/**
- * @param key key of configuration that consistent with the spring config.
- * @param defaultValue default value of configuration that <b>should not be null</b>.
- * @param classType the class type of value. <b>please use java class type</b>.
- * @param required is required <b>
- * @param description description of configuration.
- * @param handle Processing function of special parameters
+ * @param key
+ * key of configuration that consistent with the spring config.
+ * @param defaultValue
+ * default value of configuration that <b>should not be null</b>.
+ * @param classType
+ * the class type of value. <b>please use java class type</b>.
+ * @param required
+ * is required <b>
+ * @param description
+ * description of configuration.
+ * @param handle
+ * Processing function of special parameters
*/
-case class ConfigOption[T](key: String, defaultValue: T = null, required: Boolean, classType: Class[_], description: String = "", handle: String => T = null)(
- implicit
- prefix: String = "",
- prop: Properties) {
+case class ConfigOption[T](
+ key: String,
+ defaultValue: T = null,
+ required: Boolean,
+ classType: Class[_],
+ description: String = "",
+ handle: String => T = null)(implicit prefix: String = "", prop: Properties) {
private[this] lazy val fullKey = if (prefix != null && prefix.nonEmpty) s"$prefix.$key" else key
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/conf/FlinkVersion.scala b/streampark-common/src/main/scala/org/apache/streampark/common/conf/FlinkVersion.scala
index 1cb86ee1d..e2182c650 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/conf/FlinkVersion.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/conf/FlinkVersion.scala
@@ -16,6 +16,8 @@
*/
package org.apache.streampark.common.conf
+import org.apache.streampark.common.util.{CommandUtils, Logger}
+
import java.io.File
import java.net.{URL => NetURL}
import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference}
@@ -25,18 +27,15 @@ import java.util.regex.Pattern
import scala.collection.JavaConversions._
import scala.collection.mutable
-import org.apache.streampark.common.util.{CommandUtils, Logger}
-
-/**
- * @param flinkHome actual flink home that must be a readable local path
- */
+/** @param flinkHome actual flink home that must be a readable local path */
class FlinkVersion(val flinkHome: String) extends java.io.Serializable with Logger {
private[this] lazy val FLINK_VER_PATTERN = Pattern.compile("^(\\d+\\.\\d+)(\\.)?.*$")
private[this] lazy val FLINK_VERSION_PATTERN = Pattern.compile("^Version: (.*), Commit ID: (.*)$")
- private[this] lazy val FLINK_SCALA_VERSION_PATTERN = Pattern.compile("^flink-dist_(.*)-[0-9].*.jar$")
+ private[this] lazy val FLINK_SCALA_VERSION_PATTERN =
+ Pattern.compile("^flink-dist_(.*)-[0-9].*.jar$")
lazy val scalaVersion: String = {
val matcher = FLINK_SCALA_VERSION_PATTERN.matcher(flinkDistJar.getName)
@@ -54,7 +53,9 @@ class FlinkVersion(val flinkHome: String) extends java.io.Serializable with Logg
require(flinkHome != null, "[StreamPark] flinkHome must not be null.")
require(new File(flinkHome).exists(), "[StreamPark] flinkHome must be exists.")
val lib = new File(s"$flinkHome/lib")
- require(lib.exists() && lib.isDirectory, s"[StreamPark] $flinkHome/lib must be exists and must be directory.")
+ require(
+ lib.exists() && lib.isDirectory,
+ s"[StreamPark] $flinkHome/lib must be exists and must be directory.")
lib
}
@@ -62,7 +63,8 @@ class FlinkVersion(val flinkHome: String) extends java.io.Serializable with Logg
lazy val version: String = {
val flinkVersion = new AtomicReference[String]
- val cmd = List(s"java -classpath ${flinkDistJar.getAbsolutePath} org.apache.flink.client.cli.CliFrontend --version")
+ val cmd = List(
+ s"java -classpath ${flinkDistJar.getAbsolutePath} org.apache.flink.client.cli.CliFrontend --version")
val success = new AtomicBoolean(false)
val buffer = new mutable.StringBuilder
CommandUtils.execute(
@@ -77,7 +79,8 @@ class FlinkVersion(val flinkHome: String) extends java.io.Serializable with Logg
flinkVersion.set(matcher.group(1))
}
}
- })
+ }
+ )
logInfo(buffer.toString())
if (!success.get()) {
throw new IllegalStateException(s"[StreamPark] parse flink version failed. $buffer")
@@ -103,7 +106,8 @@ class FlinkVersion(val flinkHome: String) extends java.io.Serializable with Logg
case x if x.isEmpty =>
throw new IllegalArgumentException(s"[StreamPark] can no found flink-dist jar in $flinkLib")
case x if x.length > 1 =>
- throw new IllegalArgumentException(s"[StreamPark] found multiple flink-dist jar in $flinkLib")
+ throw new IllegalArgumentException(
+ s"[StreamPark] found multiple flink-dist jar in $flinkLib")
case _ =>
}
distJar.head
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/conf/InternalConfigHolder.scala b/streampark-common/src/main/scala/org/apache/streampark/common/conf/InternalConfigHolder.scala
index 893801c79..e06a900a3 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/conf/InternalConfigHolder.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/conf/InternalConfigHolder.scala
@@ -17,44 +17,37 @@
package org.apache.streampark.common.conf
+import org.apache.streampark.common.util.{Logger, SystemPropertyUtils}
import org.apache.streampark.common.util.Utils.StringCasts
+import javax.annotation.{Nonnull, Nullable}
+
import java.util
import java.util.concurrent.ConcurrentHashMap
-import javax.annotation.{Nonnull, Nullable}
+
import scala.collection.JavaConversions._
import scala.language.postfixOps
-import org.apache.streampark.common.util.{Logger, SystemPropertyUtils}
/**
- * Thread-safe configuration storage containers.
- * All configurations will be automatically initialized from the spring
- * configuration items of the same name.
+ * Thread-safe configuration storage containers. All configurations will be automatically
+ * initialized from the spring configuration items of the same name.
*/
object InternalConfigHolder extends Logger {
private val initialCapacity = 45
- /**
- * configuration values storage (key -> value)
- */
+ /** configuration values storage (key -> value) */
private val confData = new ConcurrentHashMap[String, Any](initialCapacity)
- /**
- * configuration key options storage (key -> ConfigOption)
- */
+ /** configuration key options storage (key -> ConfigOption) */
private val confOptions = new ConcurrentHashMap[String, InternalOption](initialCapacity)
- /**
- * Initialize the ConfigHub.
- */
+ /** Initialize the ConfigHub. */
{
Seq(CommonConfig, K8sFlinkConfig)
}
- /**
- * Register the ConfigOption
- */
+ /** Register the ConfigOption */
private[conf] def register(@Nonnull conf: InternalOption): Unit = {
confOptions.put(conf.key, conf)
if (conf.defaultValue != null) {
@@ -66,13 +59,12 @@ object InternalConfigHolder extends Logger {
* Get configuration value via ConfigOption.
*
* When using this api, the type must be explicitly declared and the relevant type will be
- * automatically converted to some extent.
- * 1) in scala:
- * val result: Long = ConfigHub.get(K8sFlinkConfig.sglMetricTrackTaskTimeoutSec)
- * 2) in java:
- * Long result = ConfigHub.get(K8sFlinkConfig.sglMetricTrackTaskTimeoutSec());
+ * automatically converted to some extent. 1) in scala: val result: Long =
+ * ConfigHub.get(K8sFlinkConfig.sglMetricTrackTaskTimeoutSec) 2) in java: Long result =
+ * ConfigHub.get(K8sFlinkConfig.sglMetricTrackTaskTimeoutSec());
*
- * @return return the defaultValue of ConfigOption when the value has not been overwritten.
+ * @return
+ * return the defaultValue of ConfigOption when the value has not been overwritten.
*/
@Nonnull
def get[T](@Nonnull conf: InternalOption): T = {
@@ -90,14 +82,15 @@ object InternalConfigHolder extends Logger {
* Get configuration value via key.
*
* When using this api, the type must be explicitly declared and the relevant type will be
- * automatically converted to some extent.
- * 1) in scala:
- * val result: Long = ConfigHub.get("streampark.flink-k8s.tracking.polling-task-timeout-sec.cluster-metric")
- * 2) in java:
- * Long result = ConfigHub.get("streampark.flink-k8s.tracking.polling-task-timeout-sec.cluster-metric");
+ * automatically converted to some extent. 1) in scala: val result: Long =
+ * ConfigHub.get("streampark.flink-k8s.tracking.polling-task-timeout-sec.cluster-metric") 2) in
+ * java: Long result =
+ * ConfigHub.get("streampark.flink-k8s.tracking.polling-task-timeout-sec.cluster-metric");
*
- * @throws IllegalArgumentException when the key has not been registered to ConfigHub.
- * @return return the defaultValue of ConfigOption when the value has not been overwritten.
+ * @throws IllegalArgumentException
+ * when the key has not been registered to ConfigHub.
+ * @return
+ * return the defaultValue of ConfigOption when the value has not been overwritten.
*/
@throws[IllegalArgumentException]
@Nonnull
@@ -109,7 +102,8 @@ object InternalConfigHolder extends Logger {
val config = getConfig(key)
SystemPropertyUtils.get(key) match {
case v if v != null => v.cast[T](config.classType)
- case _ => throw new IllegalArgumentException(s"config key has not been registered: $key")
+ case _ =>
+ throw new IllegalArgumentException(s"config key has not been registered: $key")
}
case conf: InternalOption => conf.defaultValue.asInstanceOf[T]
}
@@ -120,16 +114,15 @@ object InternalConfigHolder extends Logger {
/**
* Get registered ConfigOption by key.
*
- * @return nullable
+ * @return
+ * nullable
*/
@Nullable
def getConfig(key: String): InternalOption = {
confOptions.get(key)
}
- /**
- * Get keys of all registered ConfigOption.
- */
+ /** Get keys of all registered ConfigOption. */
@Nonnull
def keys(): util.Set[String] = {
val map = new util.HashMap[String, InternalOption](confOptions.size())
@@ -140,10 +133,12 @@ object InternalConfigHolder extends Logger {
/**
* Overwritten configuration value.
*
- * @param conf should not be null.
- * @param value the type of value should be same as conf.classType.
- * @throws IllegalArgumentException when the conf has not been registered,
- * or the value type is not same as conf.classType.
+ * @param conf
+ * should not be null.
+ * @param value
+ * the type of value should be same as conf.classType.
+ * @throws IllegalArgumentException
+ * when the conf has not been registered, or the value type is not same as conf.classType.
*/
@throws[IllegalArgumentException]
def set(@Nonnull conf: InternalOption, value: Any): Unit = {
@@ -161,16 +156,17 @@ object InternalConfigHolder extends Logger {
}
}
- /**
- * log the current configuration info.
- */
+ /** log the current configuration info. */
def log(): Unit = {
val configKeys = keys()
- logInfo(
- s"""registered configs:
- |ConfigHub collected configs: ${configKeys.size}
- | ${configKeys.map(key => s"$key = ${if (key.contains("password")) ConfigConst.DEFAULT_DATAMASK_STRING else get(key)}").mkString(
- "\n ")}""".stripMargin)
+ logInfo(s"""registered configs:
+ |ConfigHub collected configs: ${configKeys.size}
+ | ${configKeys
+ .map(
+ key =>
+ s"$key = ${if (key.contains("password")) ConfigConst.DEFAULT_DATAMASK_STRING
+ else get(key)}")
+ .mkString("\n ")}""".stripMargin)
}
}
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/conf/InternalOption.scala b/streampark-common/src/main/scala/org/apache/streampark/common/conf/InternalOption.scala
index e27ac0258..86816356a 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/conf/InternalOption.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/conf/InternalOption.scala
@@ -20,11 +20,19 @@ package org.apache.streampark.common.conf
/**
* Internal use of the system
*
- * @param key key of configuration that consistent with the spring config.
- * @param defaultValue default value of configuration that <b>should not be null</b>.
- * @param classType the class type of value. <b>please use java class type</b>.
- * @param description description of configuration.
+ * @param key
+ * key of configuration that consistent with the spring config.
+ * @param defaultValue
+ * default value of configuration that <b>should not be null</b>.
+ * @param classType
+ * the class type of value. <b>please use java class type</b>.
+ * @param description
+ * description of configuration.
*/
-case class InternalOption(key: String, defaultValue: Any, classType: Class[_], description: String = "") {
+case class InternalOption(
+ key: String,
+ defaultValue: Any,
+ classType: Class[_],
+ description: String = "") {
InternalConfigHolder.register(this)
}
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/conf/K8sFlinkConfig.scala b/streampark-common/src/main/scala/org/apache/streampark/common/conf/K8sFlinkConfig.scala
index c786e5862..e85b2fbcf 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/conf/K8sFlinkConfig.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/conf/K8sFlinkConfig.scala
@@ -17,44 +17,45 @@
package org.apache.streampark.common.conf
-/**
- * Flink kubernetes Configuration
- */
+/** Flink kubernetes Configuration */
object K8sFlinkConfig {
val jobStatusTrackTaskTimeoutSec: InternalOption = InternalOption(
key = "streampark.flink-k8s.tracking.polling-task-timeout-sec.job-status",
defaultValue = 120L,
classType = classOf[java.lang.Long],
- description = "run timeout seconds of single flink-k8s metrics tracking task")
+ description = "run timeout seconds of single flink-k8s metrics tracking task"
+ )
val metricTrackTaskTimeoutSec: InternalOption = InternalOption(
key = "streampark.flink-k8s.tracking.polling-task-timeout-sec.cluster-metric",
defaultValue = 120L,
classType = classOf[java.lang.Long],
- description = "run timeout seconds of single flink-k8s job status tracking task")
+ description = "run timeout seconds of single flink-k8s job status tracking task"
+ )
val jobStatueTrackTaskIntervalSec: InternalOption = InternalOption(
key = "streampark.flink-k8s.tracking.polling-interval-sec.job-status",
defaultValue = 5L,
classType = classOf[java.lang.Long],
- description = "interval seconds between two single flink-k8s metrics tracking task")
+ description = "interval seconds between two single flink-k8s metrics tracking task"
+ )
val metricTrackTaskIntervalSec: InternalOption = InternalOption(
key = "streampark.flink-k8s.tracking.polling-interval-sec.cluster-metric",
defaultValue = 5L,
classType = classOf[java.lang.Long],
- description = "interval seconds between two single flink-k8s metrics tracking task")
+ description = "interval seconds between two single flink-k8s metrics tracking task"
+ )
val silentStateJobKeepTrackingSec: InternalOption = InternalOption(
key = "streampark.flink-k8s.tracking.silent-state-keep-sec",
defaultValue = 60,
classType = classOf[java.lang.Integer],
- description = "retained tracking time for SILENT state flink tasks")
+ description = "retained tracking time for SILENT state flink tasks"
+ )
- /**
- * kubernetes default namespace
- */
+ /** kubernetes default namespace */
val DEFAULT_KUBERNETES_NAMESPACE = "default"
}
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/conf/Workspace.scala b/streampark-common/src/main/scala/org/apache/streampark/common/conf/Workspace.scala
index 57e7afc47..58ef8e2e9 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/conf/Workspace.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/conf/Workspace.scala
@@ -16,10 +16,11 @@
*/
package org.apache.streampark.common.conf
-import java.net.URI
import org.apache.streampark.common.enums.StorageType
-import org.apache.streampark.common.util.Utils.StringCasts
import org.apache.streampark.common.util.{HdfsUtils, SystemPropertyUtils}
+import org.apache.streampark.common.util.Utils.StringCasts
+
+import java.net.URI
object Workspace {
@@ -31,34 +32,22 @@ object Workspace {
private[this] lazy val localWorkspace = local.WORKSPACE
- /**
- * local build path
- */
+ /** local build path */
lazy val APP_LOCAL_DIST = s"$localWorkspace/dist"
- /**
- * dirPath of the maven local repository with built-in compilation process
- */
+ /** dirPath of the maven local repository with built-in compilation process */
lazy val MAVEN_LOCAL_PATH = s"$localWorkspace/mvnrepo"
- /**
- * local sourceCode path.(for git...)
- */
+ /** local sourceCode path.(for git...) */
lazy val PROJECT_LOCAL_PATH = s"$localWorkspace/project"
- /**
- * local log path.
- */
+ /** local log path. */
lazy val LOG_LOCAL_PATH = s"$localWorkspace/logs"
- /**
- * project build log path.
- */
+ /** project build log path. */
lazy val PROJECT_BUILD_LOG_PATH = s"$LOG_LOCAL_PATH/build_logs"
- /**
- * project archives path
- */
+ /** project archives path */
lazy val ARCHIVES_FILE_PATH = s"${remote.WORKSPACE}/historyserver/archive"
}
@@ -108,9 +97,7 @@ case class Workspace(storageType: StorageType) {
lazy val APP_CLIENT = s"$WORKSPACE/client"
- /**
- * store flink multi version support jars
- */
+ /** store flink multi version support jars */
lazy val APP_SHIMS = s"$WORKSPACE/shims"
lazy val APP_UPLOADS = s"$WORKSPACE/uploads"
@@ -123,9 +110,7 @@ case class Workspace(storageType: StorageType) {
lazy val APP_SAVEPOINTS = s"$WORKSPACE/savepoints"
- /**
- * store global public jars
- */
+ /** store global public jars */
lazy val APP_JARS = s"$WORKSPACE/jars"
}
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/enums/RestartStrategy.scala b/streampark-common/src/main/scala/org/apache/streampark/common/enums/RestartStrategy.scala
index 5de275ca2..d538e3ea8 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/enums/RestartStrategy.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/enums/RestartStrategy.scala
@@ -29,7 +29,9 @@ object RestartStrategy extends Enumeration {
else {
values.find(_.toString.replace("$minus", "-").equalsIgnoreCase(name)) match {
case Some(v) => v
- case _ => throw new IllegalArgumentException("[StreamPark] RestartStrategy must be (fixed-delay|failure-rate|none)")
+ case _ =>
+ throw new IllegalArgumentException(
+ "[StreamPark] RestartStrategy must be (fixed-delay|failure-rate|none)")
}
}
}
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/fs/FsOperator.scala b/streampark-common/src/main/scala/org/apache/streampark/common/fs/FsOperator.scala
index 909892c2d..131397df4 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/fs/FsOperator.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/fs/FsOperator.scala
@@ -45,17 +45,32 @@ abstract class FsOperator {
def mkCleanDirs(path: String): Unit
- def upload(srcPath: String, dstPath: String): Unit = upload(srcPath, dstPath, delSrc = false, overwrite = true)
-
- def copy(srcPath: String, dstPath: String): Unit = copy(srcPath, dstPath, delSrc = false, overwrite = true)
-
- def copyDir(srcPath: String, dstPath: String): Unit = copyDir(srcPath, dstPath, delSrc = false, overwrite = true)
-
- def upload(srcPath: String, dstPath: String, delSrc: Boolean = false, overwrite: Boolean = true): Unit
-
- def copy(srcPath: String, dstPath: String, delSrc: Boolean = false, overwrite: Boolean = true): Unit
-
- def copyDir(srcPath: String, dstPath: String, delSrc: Boolean = false, overwrite: Boolean = true): Unit
+ def upload(srcPath: String, dstPath: String): Unit =
+ upload(srcPath, dstPath, delSrc = false, overwrite = true)
+
+ def copy(srcPath: String, dstPath: String): Unit =
+ copy(srcPath, dstPath, delSrc = false, overwrite = true)
+
+ def copyDir(srcPath: String, dstPath: String): Unit =
+ copyDir(srcPath, dstPath, delSrc = false, overwrite = true)
+
+ def upload(
+ srcPath: String,
+ dstPath: String,
+ delSrc: Boolean = false,
+ overwrite: Boolean = true): Unit
+
+ def copy(
+ srcPath: String,
+ dstPath: String,
+ delSrc: Boolean = false,
+ overwrite: Boolean = true): Unit
+
+ def copyDir(
+ srcPath: String,
+ dstPath: String,
+ delSrc: Boolean = false,
+ overwrite: Boolean = true): Unit
def move(srcPath: String, dstPath: String): Unit
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/fs/HdfsOperator.scala b/streampark-common/src/main/scala/org/apache/streampark/common/fs/HdfsOperator.scala
index 9659ce451..042e9bd3b 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/fs/HdfsOperator.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/fs/HdfsOperator.scala
@@ -19,9 +19,7 @@ package org.apache.streampark.common.fs
import org.apache.streampark.common.util.{HdfsUtils, Logger}
-/**
- * Hadoop File System (aka HDFS) Operator
- */
+/** Hadoop File System (aka HDFS) Operator */
object HdfsOperator extends FsOperator with Logger {
override def exists(path: String): Boolean = HdfsUtils.exists(toHdfsPath(path))
@@ -30,16 +28,29 @@ object HdfsOperator extends FsOperator with Logger {
override def delete(path: String): Unit = HdfsUtils.delete(toHdfsPath(path))
- override def move(srcPath: String, dstPath: String): Unit = HdfsUtils.move(toHdfsPath(srcPath), toHdfsPath(dstPath))
+ override def move(srcPath: String, dstPath: String): Unit =
+ HdfsUtils.move(toHdfsPath(srcPath), toHdfsPath(dstPath))
override def upload(srcPath: String, dstPath: String, delSrc: Boolean, overwrite: Boolean): Unit =
HdfsUtils.upload(srcPath, toHdfsPath(dstPath), delSrc = delSrc, overwrite = overwrite)
override def copy(srcPath: String, dstPath: String, delSrc: Boolean, overwrite: Boolean): Unit =
- HdfsUtils.copyHdfs(toHdfsPath(srcPath), toHdfsPath(dstPath), delSrc = delSrc, overwrite = overwrite)
+ HdfsUtils.copyHdfs(
+ toHdfsPath(srcPath),
+ toHdfsPath(dstPath),
+ delSrc = delSrc,
+ overwrite = overwrite)
- override def copyDir(srcPath: String, dstPath: String, delSrc: Boolean, overwrite: Boolean): Unit =
- HdfsUtils.copyHdfsDir(toHdfsPath(srcPath), toHdfsPath(dstPath), delSrc = delSrc, overwrite = overwrite)
+ override def copyDir(
+ srcPath: String,
+ dstPath: String,
+ delSrc: Boolean,
+ overwrite: Boolean): Unit =
+ HdfsUtils.copyHdfsDir(
+ toHdfsPath(srcPath),
+ toHdfsPath(dstPath),
+ delSrc = delSrc,
+ overwrite = overwrite)
override def mkCleanDirs(path: String): Unit = {
delete(path)
@@ -47,7 +58,9 @@ object HdfsOperator extends FsOperator with Logger {
}
override def fileMd5(path: String): String = {
- require(path != null && path.nonEmpty, "[StreamPark] HdfsOperator.fileMd5: file must not be null.")
+ require(
+ path != null && path.nonEmpty,
+ "[StreamPark] HdfsOperator.fileMd5: file must not be null.")
HdfsUtils.fileMd5(toHdfsPath(path))
}
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/fs/LfsOperator.scala b/streampark-common/src/main/scala/org/apache/streampark/common/fs/LfsOperator.scala
index 7b37418bf..ca6d50b25 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/fs/LfsOperator.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/fs/LfsOperator.scala
@@ -17,18 +17,16 @@
package org.apache.streampark.common.fs
-import java.io.{File, FileInputStream}
+import org.apache.streampark.common.util.Logger
+import org.apache.streampark.common.util.Utils.{isAnyBank, notEmpty}
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.io.{FileUtils, IOUtils}
import org.apache.commons.lang3.StringUtils
-import org.apache.streampark.common.util.Logger
-import org.apache.streampark.common.util.Utils.{isAnyBank, notEmpty}
+import java.io.{File, FileInputStream}
-/**
- * Local File System (aka LFS) Operator
- */
+/** Local File System (aka LFS) Operator */
//noinspection DuplicatedCode
object LfsOperator extends FsOperator with Logger {
@@ -62,7 +60,11 @@ object LfsOperator extends FsOperator with Logger {
FileUtils.moveToDirectory(srcFile, dstFile, true)
}
- override def upload(srcPath: String, dstPath: String, delSrc: Boolean, overwrite: Boolean): Unit = {
+ override def upload(
+ srcPath: String,
+ dstPath: String,
+ delSrc: Boolean,
+ overwrite: Boolean): Unit = {
if (new File(srcPath).isDirectory) {
copyDir(srcPath, dstPath, delSrc, overwrite)
} else {
@@ -71,9 +73,8 @@ object LfsOperator extends FsOperator with Logger {
}
/**
- * When the suffixes of srcPath and dstPath are the same,
- * or the file names are the same, copy to the file,
- * otherwise copy to the directory.
+ * When the suffixes of srcPath and dstPath are the same, or the file names are the same, copy to
+ * the file, otherwise copy to the directory.
*/
override def copy(srcPath: String, dstPath: String, delSrc: Boolean, overwrite: Boolean): Unit = {
if (isAnyBank(srcPath, dstPath)) return
@@ -87,17 +88,20 @@ object LfsOperator extends FsOperator with Logger {
case f if f.exists() =>
if (f.isDirectory) new File(f, srcFile.getName) else f
case f =>
- require(f.getParentFile.exists(), "[StreamPark] dstPath is invalid and does not exist. Please check")
+ require(
+ f.getParentFile.exists(),
+ "[StreamPark] dstPath is invalid and does not exist. Please check")
f
}
}
require(srcFile.getCanonicalPath != dstFile.getCanonicalPath)
- val shouldCopy = if (overwrite) true;
- else {
- if (!dstFile.exists()) true else dstFile.getName != srcFile.getName
- }
+ val shouldCopy =
+ if (overwrite) true;
+ else {
+ if (!dstFile.exists()) true else dstFile.getName != srcFile.getName
+ }
if (shouldCopy) {
FileUtils.copyFile(srcFile, dstFile)
@@ -105,7 +109,11 @@ object LfsOperator extends FsOperator with Logger {
}
}
- override def copyDir(srcPath: String, dstPath: String, delSrc: Boolean, overwrite: Boolean): Unit = {
+ override def copyDir(
+ srcPath: String,
+ dstPath: String,
+ delSrc: Boolean,
+ overwrite: Boolean): Unit = {
if (isAnyBank(srcPath, dstPath)) return
val srcFile = new File(srcPath)
if (!srcFile.exists) return
@@ -123,30 +131,29 @@ object LfsOperator extends FsOperator with Logger {
}
override def fileMd5(path: String): String = {
- require(path != null && path.nonEmpty, s"[StreamPark] LFsOperator.fileMd5: file must not be null.")
+ require(
+ path != null && path.nonEmpty,
+ s"[StreamPark] LFsOperator.fileMd5: file must not be null.")
val file = new File(path)
require(file.exists, s"[StreamPark] LFsOperator.fileMd5: file must exists.")
DigestUtils.md5Hex(IOUtils.toByteArray(new FileInputStream(path)))
}
- /**
- * Force delete directory and recreate it.
- */
+ /** Force delete directory and recreate it. */
override def mkCleanDirs(path: String): Unit = {
delete(path)
mkdirs(path)
}
- /**
- * list file under directory, one level of traversal only
- */
+ /** list file under directory, one level of traversal only */
def listDir(path: String): Array[File] = {
if (path == null || path.trim.isEmpty) Array.empty
- else new File(path) match {
- case f if !f.exists => Array()
- case f if f.isFile => Array(f)
- case f => f.listFiles()
- }
+ else
+ new File(path) match {
+ case f if !f.exists => Array()
+ case f if f.isFile => Array(f)
+ case f => f.listFiles()
+ }
}
}
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/ClassLoaderUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/ClassLoaderUtils.scala
index 001d69c76..620f680e7 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/ClassLoaderUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/ClassLoaderUtils.scala
@@ -64,19 +64,25 @@ object ClassLoaderUtils extends Logger {
def loadJar(jarFilePath: String): Unit = {
val jarFile = new File(jarFilePath)
- require(jarFile.exists, s"[StreamPark] ClassLoaderUtils.loadJar: jarFilePath $jarFilePath is not exists")
- require(jarFile.isFile, s"[StreamPark] ClassLoaderUtils.loadJar: jarFilePath $jarFilePath is not file")
+ require(
+ jarFile.exists,
+ s"[StreamPark] ClassLoaderUtils.loadJar: jarFilePath $jarFilePath is not exists")
+ require(
+ jarFile.isFile,
+ s"[StreamPark] ClassLoaderUtils.loadJar: jarFilePath $jarFilePath is not file")
loadPath(jarFile.getAbsolutePath)
}
def loadJars(path: String): Unit = {
val jarDir = new File(path)
require(jarDir.exists, s"[StreamPark] ClassLoaderUtils.loadJars: jarPath $path is not exists")
- require(jarDir.isDirectory, s"[StreamPark] ClassLoaderUtils.loadJars: jarPath $path is not directory")
- require(jarDir.listFiles.length > 0, s"[StreamPark] ClassLoaderUtils.loadJars: have not jar in path:$path")
- jarDir.listFiles.foreach { x =>
- loadPath(x.getAbsolutePath)
- }
+ require(
+ jarDir.isDirectory,
+ s"[StreamPark] ClassLoaderUtils.loadJars: jarPath $path is not directory")
+ require(
+ jarDir.listFiles.length > 0,
+ s"[StreamPark] ClassLoaderUtils.loadJars: have not jar in path:$path")
+ jarDir.listFiles.foreach(x => loadPath(x.getAbsolutePath))
}
def loadResource(filepath: String): Unit = {
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/CommandUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/CommandUtils.scala
index 0d83e117c..f0a133430 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/CommandUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/CommandUtils.scala
@@ -46,9 +46,14 @@ object CommandUtils extends Logger {
}
}
- def execute(directory: String, commands: JavaIterable[String], consumer: Consumer[String]): Int = {
+ def execute(
+ directory: String,
+ commands: JavaIterable[String],
+ consumer: Consumer[String]): Int = {
Try {
- require(commands != null && commands.nonEmpty, "[StreamPark] CommandUtils.execute: commands must not be null.")
+ require(
+ commands != null && commands.nonEmpty,
+ "[StreamPark] CommandUtils.execute: commands must not be null.")
logDebug(s"Command execute:\n${commands.mkString("\n")} ")
// 1) init
@@ -63,7 +68,8 @@ object CommandUtils extends Logger {
// 2) input
def input(): Unit = {
- val out = new PrintWriter(new BufferedWriter(new OutputStreamWriter(process.getOutputStream)), true)
+ val out =
+ new PrintWriter(new BufferedWriter(new OutputStreamWriter(process.getOutputStream)), true)
// scalastyle:off println
commands.foreach(out.println)
if (!commands.last.equalsIgnoreCase("exit")) {
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/CompletableFutureUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/CompletableFutureUtils.scala
index 838c4f6c7..6b4234315 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/CompletableFutureUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/CompletableFutureUtils.scala
@@ -17,7 +17,7 @@
package org.apache.streampark.common.util
-import java.util.concurrent.{Callable, CompletableFuture, ScheduledThreadPoolExecutor, ThreadFactory, TimeUnit, TimeoutException}
+import java.util.concurrent.{Callable, CompletableFuture, ScheduledThreadPoolExecutor, ThreadFactory, TimeoutException, TimeUnit}
import java.util.function.{BiConsumer, Consumer, Function => JavaFunc}
object CompletableFutureUtils {
@@ -32,9 +32,7 @@ object CompletableFutureUtils {
}
private[this] lazy val completableDelayer: ScheduledThreadPoolExecutor = {
- val delayer = new ScheduledThreadPoolExecutor(
- 1,
- new DaemonThreadFactory())
+ val delayer = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory())
delayer.setRemoveOnCancelPolicy(true)
delayer
}
@@ -69,14 +67,20 @@ object CompletableFutureUtils {
}
/**
- * Callback handler when the future is done before timeout. Callback exceptionally when the future completed exceptionally or future isn't
- * done after timeout.
+ * Callback handler when the future is done before timeout. Callback exceptionally when the future
+ * completed exceptionally or future isn't done after timeout.
*
- * @param future The future that needs to be watched.
- * @param timeout timeout
- * @param unit timeout unit
- * @param handle The handler will be called when future complete normally before timeout.
- * @param exceptionally The exceptionally will be called when future completed exceptionally or future isn't done after timeout.
+ * @param future
+ * The future that needs to be watched.
+ * @param timeout
+ * timeout
+ * @param unit
+ * timeout unit
+ * @param handle
+ * The handler will be called when future complete normally before timeout.
+ * @param exceptionally
+ * The exceptionally will be called when future completed exceptionally or future isn't done
+ * after timeout.
*/
def runTimeout[T](
future: CompletableFuture[T],
@@ -85,32 +89,36 @@ object CompletableFutureUtils {
handle: Consumer[T],
exceptionally: Consumer[Throwable]): CompletableFuture[Unit] = {
- future.applyToEither(
- setTimeout(timeout, unit),
- new JavaFunc[T, Unit]() {
- override def apply(t: T): Unit = {
- if (handle != null) {
- handle.accept(t)
+ future
+ .applyToEither(
+ setTimeout(timeout, unit),
+ new JavaFunc[T, Unit]() {
+ override def apply(t: T): Unit = {
+ if (handle != null) {
+ handle.accept(t)
+ }
+ }
+ })
+ .exceptionally(new JavaFunc[Throwable, Unit]() {
+ override def apply(t: Throwable): Unit = {
+ if (exceptionally != null) {
+ exceptionally.accept(t)
}
}
- }).exceptionally(new JavaFunc[Throwable, Unit]() {
- override def apply(t: Throwable): Unit = {
- if (exceptionally != null) {
- exceptionally.accept(t)
- }
- }
- }).whenComplete(new BiConsumer[Unit, Throwable]() {
- override def accept(t: Unit, u: Throwable): Unit = {
- if (!future.isDone) {
- future.cancel(true)
+ })
+ .whenComplete(new BiConsumer[Unit, Throwable]() {
+ override def accept(t: Unit, u: Throwable): Unit = {
+ if (!future.isDone) {
+ future.cancel(true)
+ }
}
- }
- })
+ })
}
- def runTimeout[T](future: CompletableFuture[T],
- timeout: Long,
- unit: TimeUnit): CompletableFuture[Unit] = {
+ def runTimeout[T](
+ future: CompletableFuture[T],
+ timeout: Long,
+ unit: TimeUnit): CompletableFuture[Unit] = {
runTimeout(future, timeout, unit, null, null)
}
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/ConfigUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/ConfigUtils.scala
index d9783871f..8e6dcf671 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/ConfigUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/ConfigUtils.scala
@@ -16,31 +16,38 @@
*/
package org.apache.streampark.common.util
+import org.apache.streampark.common.conf.ConfigConst._
+
import java.util.{Map => JavaMap, Properties}
import scala.collection.JavaConversions._
import scala.collection.immutable.{Map => ScalaMap}
import scala.util.Try
-import org.apache.streampark.common.conf.ConfigConst._
-
object ConfigUtils {
- def getConf(parameter: JavaMap[String, String], prefix: String = "", addfix: String = "")(implicit alias: String = ""): Properties = {
+ def getConf(parameter: JavaMap[String, String], prefix: String = "", addfix: String = "")(implicit
+ alias: String = ""): Properties = {
val map = filterParam(parameter, prefix + alias)
val prop = new Properties()
map.filter(_._2.nonEmpty).foreach { case (k, v) => prop.put(addfix + k, v) }
prop
}
- def getHBaseConfig(parameter: JavaMap[String, String])(implicit alias: String = ""): Properties = getConf(parameter, HBASE_PREFIX, HBASE_PREFIX)
+ def getHBaseConfig(parameter: JavaMap[String, String])(implicit alias: String = ""): Properties =
+ getConf(parameter, HBASE_PREFIX, HBASE_PREFIX)
- def getInfluxConfig(parameter: JavaMap[String, String])(implicit alias: String = ""): Properties = getConf(parameter, INFLUX_PREFIX)
+ def getInfluxConfig(parameter: JavaMap[String, String])(implicit alias: String = ""): Properties =
+ getConf(parameter, INFLUX_PREFIX)
- def getKafkaSinkConf(parameter: JavaMap[String, String], topic: String = "", alias: String = ""): Properties = {
+ def getKafkaSinkConf(
+ parameter: JavaMap[String, String],
+ topic: String = "",
+ alias: String = ""): Properties = {
val prefix = KAFKA_SINK_PREFIX + alias
- val param: ScalaMap[String, String] = filterParam(parameter, if (prefix.endsWith(".")) prefix else s"$prefix.")
- if (param.isEmpty) throw new IllegalArgumentException(s"${topic} init error...")
+ val param: ScalaMap[String, String] =
+ filterParam(parameter, if (prefix.endsWith(".")) prefix else s"$prefix.")
+ if (param.isEmpty) throw new IllegalArgumentException(s"$topic init error...")
else {
val kafkaProperty = new Properties()
param.foreach(x => kafkaProperty.put(x._1, x._2.trim))
@@ -48,11 +55,13 @@ object ConfigUtils {
case SIGN_EMPTY =>
val top = kafkaProperty.getOrElse(KEY_KAFKA_TOPIC, null)
if (top == null || top.split(",|\\s+").length > 1) {
- throw new IllegalArgumentException(s"Can't find a unique topic!!!,you must be input a topic")
+ throw new IllegalArgumentException(
+ s"Can't find a unique topic!!!,you must be input a topic")
} else top
case t => t
}
- val hasTopic = !kafkaProperty.toMap.exists(x => x._1 == KEY_KAFKA_TOPIC && x._2.split(",|\\s+").toSet.contains(_topic))
+ val hasTopic = !kafkaProperty.toMap.exists(
+ x => x._1 == KEY_KAFKA_TOPIC && x._2.split(",|\\s+").toSet.contains(_topic))
if (hasTopic) {
throw new IllegalArgumentException(s"Can't find a topic of:${_topic}!!!")
} else {
@@ -79,9 +88,12 @@ object ConfigUtils {
val password = parameter.toMap.getOrDefault(s"$prefix$KEY_JDBC_PASSWORD", null)
(driver, url, user, password) match {
- case (x, y, _, _) if x == null || y == null => throw new IllegalArgumentException(s"Jdbc instance:$prefix error,[driver|url] must not be null")
+ case (x, y, _, _) if x == null || y == null =>
+ throw new IllegalArgumentException(
+ s"Jdbc instance:$prefix error,[driver|url] must not be null")
case (_, _, x, y) if (x != null && y == null) || (x == null && y != null) =>
- throw new IllegalArgumentException("Jdbc instance:" + prefix + " error, [user|password] must be all null,or all not null ")
+ throw new IllegalArgumentException(
+ "Jdbc instance:" + prefix + " error, [user|password] must be all null,or all not null ")
case _ =>
}
val param: ScalaMap[String, String] = filterParam(parameter, prefix)
@@ -93,12 +105,12 @@ object ConfigUtils {
properties
}
- private[this] def filterParam(parameter: JavaMap[String, String], fix: String): ScalaMap[String, String] = {
- parameter
- .toMap
+ private[this] def filterParam(
+ parameter: JavaMap[String, String],
+ fix: String): ScalaMap[String, String] = {
+ parameter.toMap
.filter(x => x._1.startsWith(fix) && Try(x._2 != null).getOrElse(false))
- .flatMap(x =>
- Some(x._1.substring(fix.length).replaceFirst("^\\.", "") -> x._2))
+ .flatMap(x => Some(x._1.substring(fix.length).replaceFirst("^\\.", "") -> x._2))
}
}
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/DateUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/DateUtils.scala
index 9b62ca711..8c62eebe1 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/DateUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/DateUtils.scala
@@ -21,6 +21,7 @@ import java.time.{Duration, LocalDateTime}
import java.time.format.DateTimeFormatter
import java.util._
import java.util.concurrent.TimeUnit
+
import scala.collection.mutable
import scala.util._
@@ -32,7 +33,10 @@ object DateUtils {
val fullCompact = "yyyyMMddHHmmss"
- def parse(date: String, format: String = fullFormat, timeZone: TimeZone = TimeZone.getDefault): Date = {
+ def parse(
+ date: String,
+ format: String = fullFormat,
+ timeZone: TimeZone = TimeZone.getDefault): Date = {
val df: SimpleDateFormat = new SimpleDateFormat(format)
df.setTimeZone(timeZone)
df.parse(date)
@@ -46,7 +50,9 @@ object DateUtils {
def second2Date(time: Long): Date = milliSecond2Date(time * 1000)
- def now(dateFormat: String = format_yyyyMMdd, timeZone: TimeZone = TimeZone.getDefault): String = {
+ def now(
+ dateFormat: String = format_yyyyMMdd,
+ timeZone: TimeZone = TimeZone.getDefault): String = {
val df: SimpleDateFormat = new SimpleDateFormat(dateFormat)
df.setTimeZone(timeZone)
df.format(new Date())
@@ -70,7 +76,10 @@ object DateUtils {
minuteOfDay(date) * 60 + calendar.get(Calendar.SECOND)
}
- def format(date: Date = new Date(), fmt: String = fullFormat, timeZone: TimeZone = TimeZone.getDefault): String = {
+ def format(
+ date: Date = new Date(),
+ fmt: String = fullFormat,
+ timeZone: TimeZone = TimeZone.getDefault): String = {
if (date == null) null
else {
val simpleDateFormat = new SimpleDateFormat(fmt)
@@ -79,17 +88,22 @@ object DateUtils {
}
}
- def getTime(time: String, fmt: String = fullFormat, timeZone: TimeZone = TimeZone.getDefault): Long = {
+ def getTime(
+ time: String,
+ fmt: String = fullFormat,
+ timeZone: TimeZone = TimeZone.getDefault): Long = {
val simpleDateFormat = new SimpleDateFormat(fmt)
simpleDateFormat.setTimeZone(timeZone)
Try(simpleDateFormat.parse(time).getTime)
- .filter(_ > 0).getOrElse(System.currentTimeMillis())
+ .filter(_ > 0)
+ .getOrElse(System.currentTimeMillis())
}
- /**
- * Date plus and minus
- */
- def addAndSubtract(i: Int, date: Date = new Date, timeZone: TimeZone = TimeZone.getDefault): Date = {
+ /** Date plus and minus */
+ def addAndSubtract(
+ i: Int,
+ date: Date = new Date,
+ timeZone: TimeZone = TimeZone.getDefault): Date = {
val cal = Calendar.getInstance
cal.setTimeZone(timeZone)
cal.setTime(date)
@@ -125,9 +139,7 @@ object DateUtils {
sdf.parse(localTime)
}
- /**
- * <p>Description: convert UTC time to local time</p>
- */
+ /** <p>Description: convert UTC time to local time</p> */
def utcToLocal(utcTime: String, format: String = fullFormat): Date = {
val sdf = new SimpleDateFormat(format)
sdf.setTimeZone(TimeZone.getTimeZone("UTC"))
@@ -136,7 +148,8 @@ object DateUtils {
}
/**
- * Convert duration in seconds to rich time duration format. e.g. 2 days 3 hours 4 minutes 5 seconds
+ * Convert duration in seconds to rich time duration format. e.g. 2 days 3 hours 4 minutes 5
+ * seconds
*
* @param milliseconds
* @return
@@ -162,7 +175,9 @@ object DateUtils {
builder.toString
}
- def getTimeUnit(time: String, default: (Int, TimeUnit) = (5, TimeUnit.SECONDS)): (Int, TimeUnit) = {
+ def getTimeUnit(
+ time: String,
+ default: (Int, TimeUnit) = (5, TimeUnit.SECONDS)): (Int, TimeUnit) = {
val timeUnit = time match {
case "" => null
case x: String =>
@@ -185,7 +200,8 @@ object DateUtils {
}
}
- def formatFullTime(localDateTime: LocalDateTime): String = formatFullTime(localDateTime, fullFormat)
+ def formatFullTime(localDateTime: LocalDateTime): String =
+ formatFullTime(localDateTime, fullFormat)
def formatFullTime(localDateTime: LocalDateTime, pattern: String): String = {
val dateTimeFormatter = DateTimeFormatter.ofPattern(pattern)
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/DeflaterUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/DeflaterUtils.scala
index 3c67c022f..7069135e9 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/DeflaterUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/DeflaterUtils.scala
@@ -22,9 +22,7 @@ import java.util.zip.{DataFormatException, Deflater, Inflater}
object DeflaterUtils {
- /**
- * Compress the specified text
- */
+ /** Compress the specified text */
def zipString(text: String): String = {
// compression level (0 ~ 9): low to high
// create a new deflater with the specified compression level
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/FileUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/FileUtils.scala
index dc39a1865..e9d3c68bb 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/FileUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/FileUtils.scala
@@ -20,6 +20,7 @@ import java.io._
import java.net.URL
import java.util
import java.util.Scanner
+
import scala.collection.JavaConversions._
import scala.collection.mutable
@@ -29,7 +30,7 @@ object FileUtils {
val stringBuilder = new mutable.StringBuilder
if (src == null || src.length <= 0) return null
for (i <- src.indices) {
- val v: Int = src(i) & 0xFF
+ val v: Int = src(i) & 0xff
val hv: String = Integer.toHexString(v).toUpperCase
if (hv.length < 2) {
stringBuilder.append(0)
@@ -43,10 +44,11 @@ object FileUtils {
if (input == null) {
throw new RuntimeException("The inputStream can not be null")
}
- Utils.tryWithResource(input) { in =>
- val b = new Array[Byte](4)
- in.read(b, 0, b.length)
- bytesToHexString(b)
+ Utils.tryWithResource(input) {
+ in =>
+ val b = new Array[Byte](4)
+ in.read(b, 0, b.length)
+ bytesToHexString(b)
} == "504B0304"
}
@@ -72,12 +74,16 @@ object FileUtils {
}
def exists(path: String): Unit = {
- require(path != null && path.nonEmpty && new File(path).exists(), s"[StreamPark] FileUtils.exists: file $path is not exist!")
+ require(
+ path != null && path.nonEmpty && new File(path).exists(),
+ s"[StreamPark] FileUtils.exists: file $path is not exist!")
}
def getPathFromEnv(env: String): String = {
val path = System.getenv(env)
- require(Utils.notEmpty(path), s"[StreamPark] FileUtils.getPathFromEnv: $env is not set on system env")
+ require(
+ Utils.notEmpty(path),
+ s"[StreamPark] FileUtils.getPathFromEnv: $env is not set on system env")
val file = new File(path)
require(file.exists(), s"[StreamPark] FileUtils.getPathFromEnv: $env is not exist!")
file.getAbsolutePath
@@ -85,7 +91,9 @@ object FileUtils {
def resolvePath(parent: String, child: String): String = {
val file = new File(parent, child)
- require(file.exists, s"[StreamPark] FileUtils.resolvePath: ${file.getAbsolutePath} is not exist!")
+ require(
+ file.exists,
+ s"[StreamPark] FileUtils.resolvePath: ${file.getAbsolutePath} is not exist!")
file.getAbsolutePath
}
@@ -134,7 +142,8 @@ object FileUtils {
}
}
- @throws[IOException] def readString(file: File): String = {
+ @throws[IOException]
+ def readString(file: File): String = {
require(file != null && file.isFile)
val reader = new FileReader(file)
val scanner = new Scanner(reader)
@@ -151,4 +160,3 @@ object FileUtils {
}
}
-
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/HBaseClient.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/HBaseClient.scala
index ed42ba99d..e2b36fecc 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/HBaseClient.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/HBaseClient.scala
@@ -16,16 +16,16 @@
*/
package org.apache.streampark.common.util
-import java.util.Properties
-
-import scala.collection.JavaConversions._
+import org.apache.streampark.common.conf.ConfigConst._
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.{HBaseConfiguration, TableName}
import org.apache.hadoop.hbase.client.{Connection, ConnectionFactory, Table}
import org.apache.hadoop.security.UserGroupInformation
-import org.apache.streampark.common.conf.ConfigConst._
+import java.util.Properties
+
+import scala.collection.JavaConversions._
class HBaseClient(func: () => Connection) extends Serializable {
lazy val connection: Connection = func()
@@ -39,17 +39,19 @@ object HBaseClient {
def apply(prop: Properties): HBaseClient = {
val user = prop.remove(KEY_HBASE_AUTH_USER)
prop.foreach(x => conf.set(x._1, x._2))
- new HBaseClient(() => {
- if (user != null) {
- UserGroupInformation.setConfiguration(conf)
- val remoteUser: UserGroupInformation = UserGroupInformation.createRemoteUser(user.toString)
- UserGroupInformation.setLoginUser(remoteUser)
- }
- val connection = ConnectionFactory.createConnection(conf)
- sys.addShutdownHook {
- connection.close()
- }
- connection
- })
+ new HBaseClient(
+ () => {
+ if (user != null) {
+ UserGroupInformation.setConfiguration(conf)
+ val remoteUser: UserGroupInformation =
+ UserGroupInformation.createRemoteUser(user.toString)
+ UserGroupInformation.setLoginUser(remoteUser)
+ }
+ val connection = ConnectionFactory.createConnection(conf)
+ sys.addShutdownHook {
+ connection.close()
+ }
+ connection
+ })
}
}
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/HadoopConfigUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/HadoopConfigUtils.scala
index c8d16e024..61ffd1d95 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/HadoopConfigUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/HadoopConfigUtils.scala
@@ -16,6 +16,11 @@
*/
package org.apache.streampark.common.util
+import org.apache.streampark.common.fs.LfsOperator
+
+import com.google.common.collect.Maps
+import org.apache.commons.io.{FileUtils => ApacheFileUtils}
+
import java.io.File
import java.util.{Collections, Map => JavaMap, Optional => JOption}
@@ -24,23 +29,16 @@ import scala.collection.JavaConverters._
import scala.collection.immutable.ListMap
import scala.util.{Failure, Success, Try}
-import com.google.common.collect.Maps
-import org.apache.commons.io.{FileUtils => ApacheFileUtils}
-
-import org.apache.streampark.common.fs.LfsOperator
-
-/**
- * Hadoop client configuration tools mainly for flink use.
- */
+/** Hadoop client configuration tools mainly for flink use. */
object HadoopConfigUtils {
- val HADOOP_CLIENT_CONF_FILES: Array[String] = Array("core-site.xml", "hdfs-site.xml", "yarn-site.xml")
+ val HADOOP_CLIENT_CONF_FILES: Array[String] =
+ Array("core-site.xml", "hdfs-site.xml", "yarn-site.xml")
- val HIVE_CLIENT_CONF_FILES: Array[String] = Array("core-site.xml", "hdfs-site.xml", "hive-site.xml")
+ val HIVE_CLIENT_CONF_FILES: Array[String] =
+ Array("core-site.xml", "hdfs-site.xml", "hive-site.xml")
- /**
- * Get Hadoop configuration directory path from system.
- */
+ /** Get Hadoop configuration directory path from system. */
def getSystemHadoopConfDir: Option[String] = {
Try(FileUtils.getPathFromEnv("HADOOP_CONF_DIR")) match {
case Success(p) => Some(p)
@@ -50,18 +48,17 @@ object HadoopConfigUtils {
}
}
- def getSystemHadoopConfDirAsJava: JOption[String] = JOption.ofNullable(getSystemHadoopConfDir.orNull)
+ def getSystemHadoopConfDirAsJava: JOption[String] =
+ JOption.ofNullable(getSystemHadoopConfDir.orNull)
- /**
- * Get Hive configuration directory path from system.
- */
+ /** Get Hive configuration directory path from system. */
def getSystemHiveConfDir: Option[String] = Try(FileUtils.getPathFromEnv("HIVE_CONF_DIR")).toOption
def getSystemHiveConfDirAsJava: JOption[String] = JOption.ofNullable(getSystemHiveConfDir.orNull)
/**
- * Replace host information with ip of hadoop config file or hive config file.
- * Such as core-site.xml, hdfs-site.xml, hive-site.xml.
+ * Replace host information with ip of hadoop config file or hive config file. Such as
+ * core-site.xml, hdfs-site.xml, hive-site.xml.
*/
def replaceHostWithIP(configFile: File): Unit = {
if (configFile.exists && configFile.isFile && configFile.getName.endsWith(".xml")) {
@@ -73,10 +70,10 @@ object HadoopConfigUtils {
}
}
- /**
- * Replace host information with ip of configuration files under hadoop/hive config dir.
- */
- def batchReplaceHostWithIP(configDir: File, filter: Array[String] = HADOOP_CLIENT_CONF_FILES): Unit = {
+ /** Replace host information with ip of configuration files under hadoop/hive config dir. */
+ def batchReplaceHostWithIP(
+ configDir: File,
+ filter: Array[String] = HADOOP_CLIENT_CONF_FILES): Unit = {
if (!configDir.isDirectory) {
replaceHostWithIP(configDir)
return
@@ -86,11 +83,14 @@ object HadoopConfigUtils {
return
}
configDir.listFiles
- .filter(_.isFile).filter(e => filter.contains(e.getName))
+ .filter(_.isFile)
+ .filter(e => filter.contains(e.getName))
.foreach(rewriteHostIpMapper(_, hostsMap))
}
- private[this] def rewriteHostIpMapper(configFile: File, hostsMap: ListMap[String, String]): Unit = {
+ private[this] def rewriteHostIpMapper(
+ configFile: File,
+ hostsMap: ListMap[String, String]): Unit = {
// replace the host information in the configuration content
val lines = ApacheFileUtils.readLines(configFile).map {
case line if !line.trim.startsWith("<value>") => line
@@ -107,27 +107,30 @@ object HadoopConfigUtils {
ApacheFileUtils.writeLines(configFile, lines)
}
- /**
- * Read system hadoop config to Map
- */
+ /** Read system hadoop config to Map */
def readSystemHadoopConf: JavaMap[String, String] =
getSystemHadoopConfDir
- .map(confDir =>
- LfsOperator.listDir(confDir)
- .filter(f => HADOOP_CLIENT_CONF_FILES.contains(f.getName))
- .map(f => f.getName -> ApacheFileUtils.readFileToString(f, "UTF-8"))
- .toMap.asJava)
+ .map(
+ confDir =>
+ LfsOperator
+ .listDir(confDir)
+ .filter(f => HADOOP_CLIENT_CONF_FILES.contains(f.getName))
+ .map(f => f.getName -> ApacheFileUtils.readFileToString(f, "UTF-8"))
+ .toMap
+ .asJava)
.getOrElse(Collections.emptyMap[String, String]())
- /**
- * Read system hive config to Map
- */
+ /** Read system hive config to Map */
def readSystemHiveConf: JavaMap[String, String] = {
- getSystemHiveConfDir.map(confDir =>
- LfsOperator.listDir(confDir)
- .filter(f => HIVE_CLIENT_CONF_FILES.contains(f.getName))
- .map(f => f.getName -> ApacheFileUtils.readFileToString(f, "UTF-8"))
- .toMap.asJava)
+ getSystemHiveConfDir
+ .map(
+ confDir =>
+ LfsOperator
+ .listDir(confDir)
+ .filter(f => HIVE_CLIENT_CONF_FILES.contains(f.getName))
+ .map(f => f.getName -> ApacheFileUtils.readFileToString(f, "UTF-8"))
+ .toMap
+ .asJava)
.getOrElse(Collections.emptyMap[String, String]())
}
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/HadoopUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/HadoopUtils.scala
index 63e986d97..fa6d3c267 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/HadoopUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/HadoopUtils.scala
@@ -17,15 +17,8 @@
package org.apache.streampark.common.util
-import java.io.{File, IOException}
-import java.security.PrivilegedAction
-import java.util
-import java.util.{Timer, TimerTask}
-import java.util.concurrent._
-import javax.security.auth.kerberos.KerberosTicket
-
-import scala.collection.JavaConversions._
-import scala.util.{Failure, Success, Try}
+import org.apache.streampark.common.conf.{CommonConfig, ConfigConst, InternalConfigHolder}
+import org.apache.streampark.common.conf.ConfigConst._
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang3.StringUtils
@@ -38,8 +31,16 @@ import org.apache.hadoop.yarn.api.records.ApplicationId
import org.apache.hadoop.yarn.client.api.YarnClient
import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.streampark.common.conf.{CommonConfig, ConfigConst, InternalConfigHolder}
-import org.apache.streampark.common.conf.ConfigConst._
+import javax.security.auth.kerberos.KerberosTicket
+
+import java.io.{File, IOException}
+import java.security.PrivilegedAction
+import java.util
+import java.util.{Timer, TimerTask}
+import java.util.concurrent._
+
+import scala.collection.JavaConversions._
+import scala.util.{Failure, Success, Try}
object HadoopUtils extends Logger {
@@ -57,24 +58,28 @@ object HadoopUtils extends Logger {
private[this] var tgt: KerberosTicket = _
- private lazy val hadoopUserName: String = InternalConfigHolder.get(CommonConfig.STREAMPARK_HADOOP_USER_NAME)
+ private lazy val hadoopUserName: String =
+ InternalConfigHolder.get(CommonConfig.STREAMPARK_HADOOP_USER_NAME)
- private[this] lazy val debugKerberos = kerberosConf.getOrElse(KEY_SECURITY_KERBEROS_DEBUG, "false")
+ private[this] lazy val debugKerberos =
+ kerberosConf.getOrElse(KEY_SECURITY_KERBEROS_DEBUG, "false")
- private[this] lazy val configurationCache: util.Map[String, Configuration] = new ConcurrentHashMap[String, Configuration]()
+ private[this] lazy val configurationCache: util.Map[String, Configuration] =
+ new ConcurrentHashMap[String, Configuration]()
- private[this] lazy val kerberosConf: Map[String, String] = SystemPropertyUtils.get(ConfigConst.KEY_APP_HOME, null) match {
- case null =>
- getClass.getResourceAsStream("/kerberos.yml") match {
- case x if x != null => PropertiesUtils.fromYamlFile(x)
- case _ => null
- }
- case f =>
- val file = new File(s"$f/conf/kerberos.yml")
- if (file.exists() && file.isFile) {
- PropertiesUtils.fromYamlFile(file.getAbsolutePath)
- } else null
- }
+ private[this] lazy val kerberosConf: Map[String, String] =
+ SystemPropertyUtils.get(ConfigConst.KEY_APP_HOME, null) match {
+ case null =>
+ getClass.getResourceAsStream("/kerberos.yml") match {
+ case x if x != null => PropertiesUtils.fromYamlFile(x)
+ case _ => null
+ }
+ case f =>
+ val file = new File(s"$f/conf/kerberos.yml")
+ if (file.exists() && file.isFile) {
+ PropertiesUtils.fromYamlFile(file.getAbsolutePath)
+ } else null
+ }
def getUgi(): UserGroupInformation = {
if (ugi == null) {
@@ -91,7 +96,8 @@ object HadoopUtils extends Logger {
ugi
}
- private[this] lazy val hadoopConfDir: String = Try(FileUtils.getPathFromEnv(HADOOP_CONF_DIR)) match {
+ private[this] lazy val hadoopConfDir: String = Try(
+ FileUtils.getPathFromEnv(HADOOP_CONF_DIR)) match {
case Failure(_) => FileUtils.resolvePath(FileUtils.getPathFromEnv(HADOOP_HOME), CONF_SUFFIX)
case Success(value) => value
}
@@ -114,7 +120,9 @@ object HadoopUtils extends Logger {
case TimeUnit.MINUTES => timeUnit._1 * 60 * 1000
case TimeUnit.HOURS => timeUnit._1 * 60 * 60 * 1000
case TimeUnit.DAYS => timeUnit._1 * 60 * 60 * 24 * 1000
- case _ => throw new IllegalArgumentException(s"[StreamPark] parameter:${CommonConfig.KERBEROS_TTL.key} invalided, unit options are [s|m|h|d]")
+ case _ =>
+ throw new IllegalArgumentException(
+ s"[StreamPark] parameter:${CommonConfig.KERBEROS_TTL.key} invalided, unit options are [s|m|h|d]")
}
}
}
@@ -124,7 +132,8 @@ object HadoopUtils extends Logger {
FileUtils.exists(confDir)
val hadoopConfDir = new File(confDir)
val confName = List("hdfs-default.xml", "core-site.xml", "hdfs-site.xml", "yarn-site.xml")
- val files = hadoopConfDir.listFiles().filter(x => x.isFile && confName.contains(x.getName)).toList
+ val files =
+ hadoopConfDir.listFiles().filter(x => x.isFile && confName.contains(x.getName)).toList
val conf = new HadoopConfiguration()
if (CollectionUtils.isNotEmpty(files)) {
files.foreach(x => conf.addResource(new Path(x.getAbsolutePath)))
@@ -135,12 +144,10 @@ object HadoopUtils extends Logger {
}
/**
- * <pre>
- * Note: There are two ways to load a hadoop configuration file:<br>
- * 1) Copy core-site.xml,hdfs-site.xml,yarn-site.xml of hadoop conf to the resource dir.<br>
- * 2) Automatically goes to $HADOOP_HOME/etc/hadoop to load the configuration.<br>
- * We recommend the second method, without copying the configuration files.<br>
- * </pre>
+ * <pre> Note: There are two ways to load a hadoop configuration file:<br> 1) Copy
+ * core-site.xml,hdfs-site.xml,yarn-site.xml of hadoop conf to the resource dir.<br> 2)
+ * Automatically goes to $HADOOP_HOME/etc/hadoop to load the configuration.<br> We recommend the
+ * second method, without copying the configuration files.<br> </pre>
*/
def hadoopConf: Configuration = Option(reusableConf).getOrElse {
reusableConf = getConfigurationFromHadoopConfDir(hadoopConfDir)
@@ -188,9 +195,11 @@ object HadoopUtils extends Logger {
principal.nonEmpty && keytab.nonEmpty,
s"$KEY_SECURITY_KERBEROS_PRINCIPAL and $KEY_SECURITY_KERBEROS_KEYTAB must not be empty")
- val krb5 = kerberosConf.getOrElse(
- KEY_SECURITY_KERBEROS_KRB5_CONF,
- kerberosConf.getOrElse(KEY_JAVA_SECURITY_KRB5_CONF, "")).trim
+ val krb5 = kerberosConf
+ .getOrElse(
+ KEY_SECURITY_KERBEROS_KRB5_CONF,
+ kerberosConf.getOrElse(KEY_JAVA_SECURITY_KRB5_CONF, ""))
+ .trim
if (krb5.nonEmpty) {
System.setProperty("java.security.krb5.conf", krb5)
@@ -230,11 +239,13 @@ object HadoopUtils extends Logger {
new TimerTask {
override def run(): Unit = {
closeHadoop()
- logInfo(s"Check Kerberos Tgt And reLogin From Keytab Finish:refresh time: ${DateUtils.format()}")
+ logInfo(
+ s"Check Kerberos Tgt And reLogin From Keytab Finish:refresh time: ${DateUtils.format()}")
}
},
tgtRefreshTime,
- tgtRefreshTime)
+ tgtRefreshTime
+ )
}
fs
case Failure(e) =>
@@ -255,7 +266,9 @@ object HadoopUtils extends Logger {
}
def toApplicationId(appId: String): ApplicationId = {
- require(appId != null, "[StreamPark] HadoopUtils.toApplicationId: applicationId muse not be null")
+ require(
+ appId != null,
+ "[StreamPark] HadoopUtils.toApplicationId: applicationId muse not be null")
val timestampAndId = appId.split("_")
ApplicationId.newInstance(timestampAndId(1).toLong, timestampAndId.last.toInt)
}
@@ -284,7 +297,8 @@ object HadoopUtils extends Logger {
"dfs.client.datanode-restart.timeout",
"dfs.ha.log-roll.period",
"dfs.ha.tail-edits.period",
- "dfs.datanode.bp-ready.timeout")
+ "dfs.datanode.bp-ready.timeout"
+ )
private def getHexDigits(value: String): String = {
var negative = false
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/HdfsUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/HdfsUtils.scala
index 6147285a7..5a74ab73d 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/HdfsUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/HdfsUtils.scala
@@ -16,17 +16,17 @@
*/
package org.apache.streampark.common.util
-import java.io.{ByteArrayOutputStream, FileWriter, IOException}
-import java.net.InetSocketAddress
-
-import scala.util.{Failure, Success, Try}
-
import org.apache.commons.codec.digest.DigestUtils
import org.apache.hadoop.fs._
import org.apache.hadoop.hdfs.DistributedFileSystem
import org.apache.hadoop.io.IOUtils
import org.apache.hadoop.ipc.RPC
+import java.io.{ByteArrayOutputStream, FileWriter, IOException}
+import java.net.InetSocketAddress
+
+import scala.util.{Failure, Success, Try}
+
object HdfsUtils extends Logger {
def getDefaultFS: String = HadoopUtils.hadoopConf.get(FileSystem.FS_DEFAULT_NAME_KEY)
@@ -37,28 +37,61 @@ object HdfsUtils extends Logger {
def mkdirs(path: String): Unit = HadoopUtils.hdfs.mkdirs(getPath(path))
- def copyHdfs(src: String, dst: String, delSrc: Boolean = false, overwrite: Boolean = true): Unit = {
+ def copyHdfs(
+ src: String,
+ dst: String,
+ delSrc: Boolean = false,
+ overwrite: Boolean = true): Unit = {
val srcPath = getPath(src)
val dstPath = getPath(dst)
val dstStatus = HadoopUtils.hdfs.getFileStatus(dstPath)
- val dstFinalPath = if (dstStatus.isFile) dstPath
- else {
- getPath(s"$dst/${srcPath.getName}")
- }
- FileUtil.copy(HadoopUtils.hdfs, srcPath, HadoopUtils.hdfs, dstFinalPath, delSrc, overwrite, HadoopUtils.hadoopConf)
+ val dstFinalPath =
+ if (dstStatus.isFile) dstPath
+ else {
+ getPath(s"$dst/${srcPath.getName}")
+ }
+ FileUtil.copy(
+ HadoopUtils.hdfs,
+ srcPath,
+ HadoopUtils.hdfs,
+ dstFinalPath,
+ delSrc,
+ overwrite,
+ HadoopUtils.hadoopConf)
}
- def copyHdfsDir(src: String, dst: String, delSrc: Boolean = false, overwrite: Boolean = true): Unit = {
- list(src).foreach(x => FileUtil.copy(HadoopUtils.hdfs, x, HadoopUtils.hdfs, getPath(dst), delSrc, overwrite, HadoopUtils.hadoopConf))
+ def copyHdfsDir(
+ src: String,
+ dst: String,
+ delSrc: Boolean = false,
+ overwrite: Boolean = true): Unit = {
+ list(src).foreach(
+ x =>
+ FileUtil.copy(
+ HadoopUtils.hdfs,
+ x,
+ HadoopUtils.hdfs,
+ getPath(dst),
+ delSrc,
+ overwrite,
+ HadoopUtils.hadoopConf))
}
def upload(src: String, dst: String, delSrc: Boolean = false, overwrite: Boolean = true): Unit =
HadoopUtils.hdfs.copyFromLocalFile(delSrc, overwrite, getPath(src), getPath(dst))
- def uploadMulti(src: Array[String], dst: String, delSrc: Boolean = false, overwrite: Boolean = true): Unit =
+ def uploadMulti(
+ src: Array[String],
+ dst: String,
+ delSrc: Boolean = false,
+ overwrite: Boolean = true): Unit =
HadoopUtils.hdfs.copyFromLocalFile(delSrc, overwrite, src.map(getPath), getPath(dst))
- def download(src: String, dst: String, delSrc: Boolean = false, useRawLocalFileSystem: Boolean = false): Unit =
+ def download(
+ src: String,
+ dst: String,
+ delSrc: Boolean = false,
+ useRawLocalFileSystem: Boolean = false): Unit =
HadoopUtils.hdfs.copyToLocalFile(delSrc, getPath(src), getPath(dst), useRawLocalFileSystem)
def getNameNode: String = Try(getAddressOfActive(HadoopUtils.hdfs).getHostString) match {
@@ -71,7 +104,7 @@ object HdfsUtils extends Logger {
*
* @param fileName
* @param content
- * @throws
+ * \@throws
*/
def create(fileName: String, content: String): Unit = {
val path: Path = getPath(fileName)
@@ -86,7 +119,9 @@ object HdfsUtils extends Logger {
def read(fileName: String): String = {
val path: Path = getPath(fileName)
- require(HadoopUtils.hdfs.exists(path) && !HadoopUtils.hdfs.isDirectory(path), s"[StreamPark] HdfsUtils.read: path($fileName) not exists or isDirectory ")
+ require(
+ HadoopUtils.hdfs.exists(path) && !HadoopUtils.hdfs.isDirectory(path),
+ s"[StreamPark] HdfsUtils.read: path($fileName) not exists or isDirectory ")
val in = HadoopUtils.hdfs.open(path)
val out = new ByteArrayOutputStream()
IOUtils.copyBytes(in, out, 4096, false)
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/HostsUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/HostsUtils.scala
index c2ec2dc91..871165c07 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/HostsUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/HostsUtils.scala
@@ -17,44 +17,40 @@
package org.apache.streampark.common.util
+import io.netty.resolver.HostsFileParser
+
import java.net.InetAddress
import java.util.{Map => JavaMap}
import scala.collection.JavaConversions._
import scala.collection.immutable.ListMap
-import io.netty.resolver.HostsFileParser
-
object HostsUtils {
/**
- * Get hosts info from system, entry of return Map [hostname -> ipv4].
- * The elements are sorted in reverse order by the length of the hostname.
+ * Get hosts info from system, entry of return Map [hostname -> ipv4]. The elements are sorted in
+ * reverse order by the length of the hostname.
*/
def getSortSystemHosts: ListMap[String, String] = {
val ipMap = HostsFileParser.parse.inet4Entries.map(e => e._1 -> e._2.getHostAddress)
ListMap(ipMap.toSeq.sortWith(_._1.length > _._1.length): _*)
}
- /**
- * Get hosts info from system, entry of return Map [hostname -> ipv4].
- * Scala api.
- */
+ /** Get hosts info from system, entry of return Map [hostname -> ipv4]. Scala api. */
def getSystemHosts(excludeLocalHost: Boolean = false): Map[String, String] = {
var map = HostsFileParser.parse.inet4Entries.map(e => e._1 -> e._2.getHostAddress).toMap
if (excludeLocalHost) {
val localHostName = InetAddress.getLocalHost.getHostName
- map = map.filter(!_._1.equals("localhost"))
+ map = map
+ .filter(!_._1.equals("localhost"))
.filter(!_._1.equals(localHostName))
.filter(!_._2.equals("127.0.0.1"))
}
map
}
- /**
- * Get hosts info from system, entry of return Map [hostname -> ipv4].
- * Java api.
- */
- def getSystemHostsAsJava(excludeLocalHost: Boolean): JavaMap[String, String] = getSystemHosts(excludeLocalHost)
+ /** Get hosts info from system, entry of return Map [hostname -> ipv4]. Java api. */
+ def getSystemHostsAsJava(excludeLocalHost: Boolean): JavaMap[String, String] = getSystemHosts(
+ excludeLocalHost)
}
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/HttpClientUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/HttpClientUtils.scala
index c000d6991..5e2e7d5c5 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/HttpClientUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/HttpClientUtils.scala
@@ -16,12 +16,6 @@
*/
package org.apache.streampark.common.util
-import java.nio.charset.{Charset, StandardCharsets}
-import java.security.Principal
-import java.util
-
-import scala.collection.JavaConversions._
-
import org.apache.http.NameValuePair
import org.apache.http.auth.{AuthSchemeProvider, AuthScope, Credentials}
import org.apache.http.client.config.{AuthSchemes, RequestConfig}
@@ -36,6 +30,12 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager
import org.apache.http.message.BasicNameValuePair
import org.apache.http.util.EntityUtils
+import java.nio.charset.{Charset, StandardCharsets}
+import java.security.Principal
+import java.util
+
+import scala.collection.JavaConversions._
+
object HttpClientUtils {
private[this] val defaultChart: Charset = StandardCharsets.UTF_8
@@ -47,12 +47,13 @@ object HttpClientUtils {
connectionManager
}
- /**
- * Get HttpClient with connection manager
- */
+ /** Get HttpClient with connection manager */
private[this] def getHttpClient = HttpClients.custom.setConnectionManager(connectionManager).build
- private[this] def getHttpGet(url: String, params: util.Map[String, AnyRef] = null, config: RequestConfig = null): HttpGet = {
+ private[this] def getHttpGet(
+ url: String,
+ params: util.Map[String, AnyRef] = null,
+ config: RequestConfig = null): HttpGet = {
val httpGet = params match {
case null => new HttpGet(url)
case _ =>
@@ -74,11 +75,18 @@ object HttpClientUtils {
getHttpResult(getHttpGet(url, null, config))
}
- def httpGetRequest(url: String, config: RequestConfig, params: util.Map[String, AnyRef]): String = {
+ def httpGetRequest(
+ url: String,
+ config: RequestConfig,
+ params: util.Map[String, AnyRef]): String = {
getHttpResult(getHttpGet(url, params, config))
}
- def httpGetRequest(url: String, config: RequestConfig, headers: util.Map[String, AnyRef], params: util.Map[String, AnyRef]): String = {
+ def httpGetRequest(
+ url: String,
+ config: RequestConfig,
+ headers: util.Map[String, AnyRef],
+ params: util.Map[String, AnyRef]): String = {
val httpGet = getHttpGet(url, params, config)
headers.entrySet.foreach(p => httpGet.addHeader(p.getKey, String.valueOf(p.getValue)))
getHttpResult(httpGet)
@@ -108,17 +116,26 @@ object HttpClientUtils {
def httpPostRequest(url: String, params: String): String = httpRequest(new HttpPost(url), params)
- def httpPatchRequest(url: String, params: String): String = httpRequest(new HttpPatch(url), params)
+ def httpPatchRequest(url: String, params: String): String =
+ httpRequest(new HttpPatch(url), params)
- def httpPostRequest(url: String, params: util.Map[String, AnyRef], headers: util.Map[String, AnyRef] = Map.empty[String, AnyRef]): String = {
+ def httpPostRequest(
+ url: String,
+ params: util.Map[String, AnyRef],
+ headers: util.Map[String, AnyRef] = Map.empty[String, AnyRef]): String = {
httpRequest(new HttpPost(url), headers, params)
}
- def httpPatchRequest(url: String, params: util.Map[String, AnyRef], headers: util.Map[String, AnyRef] = Map.empty[String, AnyRef]): String = {
+ def httpPatchRequest(
+ url: String,
+ params: util.Map[String, AnyRef],
+ headers: util.Map[String, AnyRef] = Map.empty[String, AnyRef]): String = {
httpRequest(new HttpPatch(url), headers, params)
}
- private[this] def httpRequest(httpEntity: HttpEntityEnclosingRequestBase, params: String): String = {
+ private[this] def httpRequest(
+ httpEntity: HttpEntityEnclosingRequestBase,
+ params: String): String = {
val entity = new StringEntity(params, defaultChart)
entity.setContentEncoding("UTF-8")
entity.setContentType("application/json")
@@ -126,15 +143,20 @@ object HttpClientUtils {
getHttpResult(httpEntity)
}
- private[this] def httpRequest(httpPatch: HttpEntityEnclosingRequestBase, headers: util.Map[String, AnyRef], params: util.Map[String, AnyRef]) = {
+ private[this] def httpRequest(
+ httpPatch: HttpEntityEnclosingRequestBase,
+ headers: util.Map[String, AnyRef],
+ params: util.Map[String, AnyRef]) = {
headers.entrySet.foreach(p => httpPatch.addHeader(p.getKey, String.valueOf(p.getValue)))
httpPatch.setEntity(new UrlEncodedFormEntity(paramsToNameValuePairs(params), defaultChart))
getHttpResult(httpPatch)
}
- private[this] def paramsToNameValuePairs(params: util.Map[String, AnyRef]): util.List[NameValuePair] = {
+ private[this] def paramsToNameValuePairs(
+ params: util.Map[String, AnyRef]): util.List[NameValuePair] = {
val pairs = new util.ArrayList[NameValuePair]
- params.entrySet.foreach(p => pairs.add(new BasicNameValuePair(p.getKey, String.valueOf(p.getValue))))
+ params.entrySet.foreach(
+ p => pairs.add(new BasicNameValuePair(p.getKey, String.valueOf(p.getValue))))
pairs
}
@@ -149,24 +171,26 @@ object HttpClientUtils {
override def getPassword: String = null
})
- val authSchemeRegistry = RegistryBuilder.create[AuthSchemeProvider]
- .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true)).build
+ val authSchemeRegistry = RegistryBuilder
+ .create[AuthSchemeProvider]
+ .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true))
+ .build
- HttpClientBuilder.create()
+ HttpClientBuilder
+ .create()
.setDefaultAuthSchemeRegistry(authSchemeRegistry)
.setDefaultCredentialsProvider(credentialsProvider)
- .setConnectionManager(connectionManager).build()
+ .setConnectionManager(connectionManager)
+ .build()
}
- getHttpResult(
- getHttpGet(url, null, config),
- getHttpAuthClient)
+ getHttpResult(getHttpGet(url, null, config), getHttpAuthClient)
}
- /**
- * process http request
- */
- private[this] def getHttpResult(request: HttpRequestBase, httpClient: CloseableHttpClient = getHttpClient): String = {
+ /** process http request */
+ private[this] def getHttpResult(
+ request: HttpRequestBase,
+ httpClient: CloseableHttpClient = getHttpClient): String = {
try {
val response = httpClient.execute(request)
val entity = response.getEntity
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/JdbcUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/JdbcUtils.scala
index fa64679ce..2186b8e8d 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/JdbcUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/JdbcUtils.scala
@@ -16,6 +16,10 @@
*/
package org.apache.streampark.common.util
+import org.apache.streampark.common.conf.ConfigConst._
+
+import com.zaxxer.hikari.{HikariConfig, HikariDataSource}
+
import java.sql.{Connection, ResultSet, Statement}
import java.util.Properties
import java.util.concurrent.ConcurrentHashMap
@@ -26,24 +30,20 @@ import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
import scala.util.Try
-import com.zaxxer.hikari.{HikariConfig, HikariDataSource}
-
-import org.apache.streampark.common.conf.ConfigConst._
-
/**
- * Based on the hikari connection pool implementation. Support multiple data sources,
- * note that all modifications and additions are automatically committed transactions.
+ * Based on the hikari connection pool implementation. Support multiple data sources, note that all
+ * modifications and additions are automatically committed transactions.
*/
object JdbcUtils {
- private val lockMap: mutable.Map[String, ReentrantLock] = new ConcurrentHashMap[String, ReentrantLock]
+ private val lockMap: mutable.Map[String, ReentrantLock] =
+ new ConcurrentHashMap[String, ReentrantLock]
private[this] val dataSourceHolder = new ConcurrentHashMap[String, HikariDataSource]
- /**
- * Wrap all the fields of a query row of data into a List[Map]
- */
- def select(sql: String, func: ResultSet => Unit = null)(implicit jdbcConfig: Properties): List[Map[String, _]] = {
+ /** Wrap all the fields of a query row of data into a List[Map] */
+ def select(sql: String, func: ResultSet => Unit = null)(implicit
+ jdbcConfig: Properties): List[Map[String, _]] = {
if (Try(sql.isEmpty).getOrElse(false)) List.empty
else {
val conn = getConnection(jdbcConfig)
@@ -78,7 +78,8 @@ object JdbcUtils {
}
}
- def count(sql: String)(implicit jdbcConfig: Properties): Long = unique(sql).head._2.toString.toLong
+ def count(sql: String)(implicit jdbcConfig: Properties): Long = unique(
+ sql).head._2.toString.toLong
def count(conn: Connection, sql: String): Long = unique(conn, sql).head._2.toString.toLong
@@ -93,16 +94,19 @@ object JdbcUtils {
try {
var index: Int = 0
val batchSize = 1000
- sql.map(x => {
- prepStat.addBatch(x)
- index += 1
- if (index > 0 && index % batchSize == 0) {
- val count = prepStat.executeBatch().sum
- conn.commit()
- prepStat.clearBatch()
- count
- } else 0
- }).sum + prepStat.executeBatch().sum
+ sql
+ .map(
+ x => {
+ prepStat.addBatch(x)
+ index += 1
+ if (index > 0 && index % batchSize == 0) {
+ val count = prepStat.executeBatch().sum
+ conn.commit()
+ prepStat.clearBatch()
+ count
+ } else 0
+ })
+ .sum + prepStat.executeBatch().sum
} catch {
case ex: Exception =>
ex.printStackTrace()
@@ -114,13 +118,16 @@ object JdbcUtils {
}
}
- def update(sql: String)(implicit jdbcConfig: Properties): Int = update(getConnection(jdbcConfig), sql)
+ def update(sql: String)(implicit jdbcConfig: Properties): Int =
+ update(getConnection(jdbcConfig), sql)
/**
- * Used to execute INSERT, UPDATE, or DELETE statements and SQL DDL statements, such as CREATE TABLE and DROP TABLE.
- * The effect of an INSERT, UPDATE, or DELETE statement is to modify one or more columns in zero or more rows of a table.
- * The return value of executeUpdate is an integer (int) indicating the number of rows affected (i.e., the update count).
- * For statements such as CREATE TABLE or DROP TABLE that do not operate on rows, the return value of executeUpdate is always zero
+ * Used to execute INSERT, UPDATE, or DELETE statements and SQL DDL statements, such as CREATE
+ * TABLE and DROP TABLE. The effect of an INSERT, UPDATE, or DELETE statement is to modify one or
+ * more columns in zero or more rows of a table. The return value of executeUpdate is an integer
+ * (int) indicating the number of rows affected (i.e., the update count). For statements such as
+ * CREATE TABLE or DROP TABLE that do not operate on rows, the return value of executeUpdate is
+ * always zero
*/
def update(conn: Connection, sql: String): Int = {
var statement: Statement = null
@@ -136,11 +143,10 @@ object JdbcUtils {
}
}
- def unique(sql: String)(implicit jdbcConfig: Properties): Map[String, _] = unique(getConnection(jdbcConfig), sql)
+ def unique(sql: String)(implicit jdbcConfig: Properties): Map[String, _] =
+ unique(getConnection(jdbcConfig), sql)
- /**
- * The query returns one row of records
- */
+ /** The query returns one row of records */
def unique(conn: Connection, sql: String): Map[String, _] = {
var stmt: Statement = null
var result: ResultSet = null
@@ -168,15 +174,16 @@ object JdbcUtils {
}
/**
- * Note about execute:
- * It can be used to execute any SQL statement, returns a boolean value indicating whether the execution
- * of the SQL statement returned a ResultSet. Returns true if the first result after execution is a
- * ResultSet, otherwise it returns false. However, it is cumbersome to execute SQL statements. Usually
- * we don't need to use the execute method to execute SQL statements,but it is more appropriate to use
- * executeQuery or executeUpdate. But if you don't know the type of the SQL statement,you can only use
- * the execute method to execute the SQL statement.
+ * Note about execute: It can be used to execute any SQL statement, returns a boolean value
+ * indicating whether the execution of the SQL statement returned a ResultSet. Returns true if the
+ * first result after execution is a ResultSet, otherwise it returns false. However, it is
+ * cumbersome to execute SQL statements. Usually we don't need to use the execute method to
+ * execute SQL statements,but it is more appropriate to use executeQuery or executeUpdate. But if
+ * you don't know the type of the SQL statement,you can only use the execute method to execute the
+ * SQL statement.
*/
- def execute(sql: String)(implicit jdbcConfig: Properties): Boolean = execute(getConnection(jdbcConfig), sql)
+ def execute(sql: String)(implicit jdbcConfig: Properties): Boolean =
+ execute(getConnection(jdbcConfig), sql)
def execute(conn: Connection, sql: String): Boolean = {
var stmt: Statement = null
@@ -192,9 +199,7 @@ object JdbcUtils {
}
}
- /**
- * Get connection with HikariDataSource
- */
+ /** Get connection with HikariDataSource */
def getConnection(prop: Properties): Connection = {
val alias = prop(KEY_ALIAS)
val lock = lockMap.getOrElseUpdate(alias, new ReentrantLock())
@@ -203,36 +208,46 @@ object JdbcUtils {
val ds: HikariDataSource = Try(Option(dataSourceHolder(alias))).getOrElse(None) match {
case None =>
val jdbcConfig = new HikariConfig()
- prop.filter(x => x._1 != KEY_ALIAS && x._1 != KEY_SEMANTIC).foreach(x => {
- Try(Option(jdbcConfig.getClass.getDeclaredField(x._1))).getOrElse(None) match {
- case Some(field) =>
- field.setAccessible(true)
- field.getType.getSimpleName match {
- case "String" => field.set(jdbcConfig, x._2.asInstanceOf[Object])
- case "int" => field.set(jdbcConfig, x._2.toInt.asInstanceOf[Object])
- case "long" => field.set(jdbcConfig, x._2.toLong.asInstanceOf[Object])
- case "boolean" => field.set(jdbcConfig, x._2.toBoolean.asInstanceOf[Object])
- case _ =>
- }
- case None =>
- val setMethod = s"set${x._1.substring(0, 1).toUpperCase}${x._1.substring(1)}"
- val method = Try(jdbcConfig.getClass.getMethods.filter(_.getName == setMethod).filter(_.getParameterCount == 1).head).getOrElse(null)
- method match {
- case m if m != null =>
- m.setAccessible(true)
- m.getParameterTypes.head.getSimpleName match {
- case "String" => m.invoke(jdbcConfig, Seq(x._2.asInstanceOf[Object]): _*)
- case "int" => m.invoke(jdbcConfig, Seq(x._2.toInt.asInstanceOf[Object]): _*)
- case "long" => m.invoke(jdbcConfig, Seq(x._2.toLong.asInstanceOf[Object]): _*)
- case "boolean" => m.invoke(jdbcConfig, Seq(x._2.toBoolean.asInstanceOf[Object]): _*)
+ prop
+ .filter(x => x._1 != KEY_ALIAS && x._1 != KEY_SEMANTIC)
+ .foreach(
+ x => {
+ Try(Option(jdbcConfig.getClass.getDeclaredField(x._1))).getOrElse(None) match {
+ case Some(field) =>
+ field.setAccessible(true)
+ field.getType.getSimpleName match {
+ case "String" => field.set(jdbcConfig, x._2.asInstanceOf[Object])
+ case "int" => field.set(jdbcConfig, x._2.toInt.asInstanceOf[Object])
+ case "long" => field.set(jdbcConfig, x._2.toLong.asInstanceOf[Object])
+ case "boolean" => field.set(jdbcConfig, x._2.toBoolean.asInstanceOf[Object])
case _ =>
}
- case null =>
- throw new IllegalArgumentException(
- s"jdbcConfig error,property:${x._1} invalid,please see more properties jdbcConfig https://github.com/brettwooldridge/HikariCP")
+ case None =>
+ val setMethod = s"set${x._1.substring(0, 1).toUpperCase}${x._1.substring(1)}"
+ val method = Try(
+ jdbcConfig.getClass.getMethods
+ .filter(_.getName == setMethod)
+ .filter(_.getParameterCount == 1)
+ .head).getOrElse(null)
+ method match {
+ case m if m != null =>
+ m.setAccessible(true)
+ m.getParameterTypes.head.getSimpleName match {
+ case "String" => m.invoke(jdbcConfig, Seq(x._2.asInstanceOf[Object]): _*)
+ case "int" =>
+ m.invoke(jdbcConfig, Seq(x._2.toInt.asInstanceOf[Object]): _*)
+ case "long" =>
+ m.invoke(jdbcConfig, Seq(x._2.toLong.asInstanceOf[Object]): _*)
+ case "boolean" =>
+ m.invoke(jdbcConfig, Seq(x._2.toBoolean.asInstanceOf[Object]): _*)
+ case _ =>
+ }
+ case null =>
+ throw new IllegalArgumentException(
+ s"jdbcConfig error,property:${x._1} invalid,please see more properties jdbcConfig https://github.com/brettwooldridge/HikariCP")
+ }
}
- }
- })
+ })
val ds = new HikariDataSource(jdbcConfig)
dataSourceHolder += alias -> ds
ds
@@ -244,8 +259,10 @@ object JdbcUtils {
}
}
- private[this] def createStatement(conn: Connection): Statement = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)
+ private[this] def createStatement(conn: Connection): Statement =
+ conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)
- def close(closeable: AutoCloseable*): Unit = Try(closeable.filter(x => x != null).foreach(_.close()))
+ def close(closeable: AutoCloseable*): Unit = Try(
+ closeable.filter(x => x != null).foreach(_.close()))
}
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/JsonUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/JsonUtils.scala
index 7f24728ea..3917f3d2d 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/JsonUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/JsonUtils.scala
@@ -16,14 +16,14 @@
*/
package org.apache.streampark.common.util
-import java.text.SimpleDateFormat
-
-import scala.reflect.ClassTag
-
import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper, SerializationFeature}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
+import java.text.SimpleDateFormat
+
+import scala.reflect.ClassTag
+
object JsonUtils extends Serializable {
private val mapper = new ObjectMapper()
@@ -51,7 +51,8 @@ object JsonUtils extends Serializable {
}
}
- def read[T](obj: AnyRef)(implicit classTag: ClassTag[T]): T = this.read(obj, classTag.runtimeClass).asInstanceOf[T]
+ def read[T](obj: AnyRef)(implicit classTag: ClassTag[T]): T =
+ this.read(obj, classTag.runtimeClass).asInstanceOf[T]
def write(obj: AnyRef): String = mapper.writeValueAsString(obj)
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/Logger.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/Logger.scala
index dbfd7b982..956eefd6c 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/Logger.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/Logger.scala
@@ -18,18 +18,18 @@ package org.apache.streampark.common.util
import org.apache.streampark.shaded.ch.qos.logback.classic.LoggerContext
import org.apache.streampark.shaded.ch.qos.logback.classic.joran.JoranConfigurator
-import org.apache.streampark.shaded.ch.qos.logback.classic.util.ContextSelectorStaticBinder
import org.apache.streampark.shaded.ch.qos.logback.classic.util.{ContextInitializer => LogBackContextInitializer}
+import org.apache.streampark.shaded.ch.qos.logback.classic.util.ContextSelectorStaticBinder
import org.apache.streampark.shaded.ch.qos.logback.core.{CoreConstants, LogbackException}
import org.apache.streampark.shaded.ch.qos.logback.core.status.StatusUtil
import org.apache.streampark.shaded.ch.qos.logback.core.util.StatusPrinter
-
import org.apache.streampark.shaded.org.slf4j.{ILoggerFactory, Logger => Slf4JLogger}
import org.apache.streampark.shaded.org.slf4j.spi.LoggerFactoryBinder
import java.io.{ByteArrayInputStream, File}
import java.net.URL
import java.nio.charset.StandardCharsets
+
import scala.util.Try
trait Logger {
@@ -99,14 +99,15 @@ private[this] object LoggerFactory extends LoggerFactoryBinder {
val defaultLoggerContext = new LoggerContext
Try(new ContextInitializer(defaultLoggerContext).autoConfig())
- .recover[Unit]({ case e =>
- val msg = "Failed to auto configure default logger context"
- // scalastyle:off println
- System.err.println(msg)
- // scalastyle:off println
- System.err.println("Reported exception:")
- e.printStackTrace()
- })
+ .recover[Unit] {
+ case e =>
+ val msg = "Failed to auto configure default logger context"
+ // scalastyle:off println
+ System.err.println(msg)
+ // scalastyle:off println
+ System.err.println("Reported exception:")
+ e.printStackTrace()
+ }
if (!StatusUtil.contextHasStatusListener(defaultLoggerContext)) {
StatusPrinter.printInCaseOfErrorsOrWarnings(defaultLoggerContext)
@@ -119,14 +120,16 @@ private[this] object LoggerFactory extends LoggerFactoryBinder {
override def getLoggerFactory: ILoggerFactory = {
if (contextSelectorBinder.getContextSelector == null) {
- throw new IllegalStateException("contextSelector cannot be null. See also " + CoreConstants.CODES_URL + "#null_CS")
+ throw new IllegalStateException(
+ "contextSelector cannot be null. See also " + CoreConstants.CODES_URL + "#null_CS")
}
contextSelectorBinder.getContextSelector.getLoggerContext
}
override def getLoggerFactoryClassStr: String = contextSelectorBinder.getClass.getName
- private class ContextInitializer(loggerContext: LoggerContext) extends LogBackContextInitializer(loggerContext) {
+ private class ContextInitializer(loggerContext: LoggerContext)
+ extends LogBackContextInitializer(loggerContext) {
val shadedPackage = "org.apache.streampark.shaded"
@@ -136,23 +139,20 @@ private[this] object LoggerFactory extends LoggerFactoryBinder {
if (path.endsWith("xml")) {
val configurator = new JoranConfigurator()
configurator.setContext(loggerContext)
- val text = FileUtils.readString(new File(path))
- .replaceAll(
- "ch.qos.logback",
- s"$shadedPackage.ch.qos.logback")
- .replaceAll(
- "org.slf4j",
- s"$shadedPackage.org.slf4j")
+ val text = FileUtils
+ .readString(new File(path))
+ .replaceAll("ch.qos.logback", s"$shadedPackage.ch.qos.logback")
+ .replaceAll("org.slf4j", s"$shadedPackage.org.slf4j")
val input = new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8))
configurator.doConfigure(input)
- } else throw {
- new LogbackException("Unexpected filename extension of file [" + url.toString + "]. Should be .xml")
- }
+ } else
+ throw {
+ new LogbackException(
+ "Unexpected filename extension of file [" + url.toString + "]. Should be .xml")
+ }
}
}
}
-
-
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/MongoConfig.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/MongoConfig.scala
index 78ac889c1..bfbdcbb2d 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/MongoConfig.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/MongoConfig.scala
@@ -16,13 +16,13 @@
*/
package org.apache.streampark.common.util
-import java.util.Properties
-
-import scala.collection.JavaConversions._
+import org.apache.streampark.common.conf.ConfigConst._
import com.mongodb._
-import org.apache.streampark.common.conf.ConfigConst._
+import java.util.Properties
+
+import scala.collection.JavaConversions._
object MongoConfig {
@@ -34,7 +34,8 @@ object MongoConfig {
val password = "password"
val min_connections_per_host = "min-connections-per-host"
val max_connections_per_host = "max-connections-per-host"
- val threads_allowed_to_block_for_connection_multiplier = "threads-allowed-to-block-for-connection-multiplier"
+ val threads_allowed_to_block_for_connection_multiplier =
+ "threads-allowed-to-block-for-connection-multiplier"
val server_selection_timeout = "server-selection-timeout"
val max_wait_time = "max-wait-time"
val max_connection_idel_time = "max-connection-idel-time"
@@ -59,10 +60,14 @@ object MongoConfig {
def getProperties(properties: Properties)(implicit alias: String = ""): Properties = {
val prop = new Properties()
- properties.filter(_._1.startsWith(MONGO_PREFIX)).filter(_._2.nonEmpty).map(x => {
- val k = x._1.replaceAll(s"$MONGO_PREFIX$alias", "").replaceFirst("^\\.", "")
- prop.put(k, x._2.trim)
- })
+ properties
+ .filter(_._1.startsWith(MONGO_PREFIX))
+ .filter(_._2.nonEmpty)
+ .map(
+ x => {
+ val k = x._1.replaceAll(s"$MONGO_PREFIX$alias", "").replaceFirst("^\\.", "")
+ prop.put(k, x._2.trim)
+ })
prop
}
@@ -84,7 +89,8 @@ object MongoConfig {
builder.requiredReplicaSetName(mongoParam(replica_set))
}
if (mongoParam.containsKey(threads_allowed_to_block_for_connection_multiplier)) {
- builder.threadsAllowedToBlockForConnectionMultiplier(mongoParam(threads_allowed_to_block_for_connection_multiplier).toInt)
+ builder.threadsAllowedToBlockForConnectionMultiplier(
+ mongoParam(threads_allowed_to_block_for_connection_multiplier).toInt)
}
if (mongoParam.containsKey(server_selection_timeout)) {
builder.serverSelectionTimeout(mongoParam(server_selection_timeout).toInt)
@@ -129,14 +135,19 @@ object MongoConfig {
builder.localThreshold(mongoParam(local_threshold).toInt)
}
val mongoClientOptions = builder.build
- val serverAddresses = mongoParam(address).split(",").map(x => {
- val hostAndPort = x.split(":")
- val host = hostAndPort.head
- val port = hostAndPort(1).toInt
- new ServerAddress(host, port)
- })
+ val serverAddresses = mongoParam(address)
+ .split(",")
+ .map(
+ x => {
+ val hostAndPort = x.split(":")
+ val host = hostAndPort.head
+ val port = hostAndPort(1).toInt
+ new ServerAddress(host, port)
+ })
if (mongoParam.containsKey(username)) {
- val db = if (mongoParam.containsKey(authentication_database)) mongoParam(authentication_database) else mongoParam(database)
+ val db =
+ if (mongoParam.containsKey(authentication_database)) mongoParam(authentication_database)
+ else mongoParam(database)
val mongoCredential = MongoCredential.createScramSha1Credential(
mongoParam(username),
db,
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/PropertiesUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/PropertiesUtils.scala
index 5fb0907c0..75ab3ceff 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/PropertiesUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/PropertiesUtils.scala
@@ -16,19 +16,21 @@
*/
package org.apache.streampark.common.util
-import java.io._
-import java.util.{Properties, Scanner, HashMap => JavaMap, LinkedHashMap => JavaLinkedMap}
-import java.util.concurrent.atomic.AtomicInteger
-import scala.collection.JavaConversions._
-import scala.collection.JavaConverters._
-import scala.collection.mutable.{Map => MutableMap}
import com.typesafe.config.ConfigFactory
import org.apache.commons.lang3.StringUtils
import org.yaml.snakeyaml.Yaml
-import java.util.regex.Pattern
import javax.annotation.Nonnull
+
+import java.io._
+import java.util.{HashMap => JavaMap, LinkedHashMap => JavaLinkedMap, Properties, Scanner}
+import java.util.concurrent.atomic.AtomicInteger
+import java.util.regex.Pattern
+
+import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable
+import scala.collection.mutable.{Map => MutableMap}
object PropertiesUtils extends Logger {
@@ -51,15 +53,22 @@ object PropertiesUtils extends Logger {
buffer.toString()
}
- private[this] def eachAppendYamlItem(prefix: String, k: String, v: Any, proper: collection.mutable.Map[String, String]): Map[String, String] = {
+ private[this] def eachAppendYamlItem(
+ prefix: String,
+ k: String,
+ v: Any,
+ proper: collection.mutable.Map[String, String]): Map[String, String] = {
v match {
case map: JavaLinkedMap[String, Any] =>
- map.flatMap(x => {
- prefix match {
- case "" => eachAppendYamlItem(k, x._1, x._2, proper)
- case other => eachAppendYamlItem(s"$other.$k", x._1, x._2, proper)
- }
- }).toMap
+ map
+ .flatMap(
+ x => {
+ prefix match {
+ case "" => eachAppendYamlItem(k, x._1, x._2, proper)
+ case other => eachAppendYamlItem(s"$other.$k", x._1, x._2, proper)
+ }
+ })
+ .toMap
case text =>
val value = text match {
case null => ""
@@ -79,9 +88,11 @@ object PropertiesUtils extends Logger {
new Yaml()
.load(text)
.asInstanceOf[java.util.Map[String, Map[String, Any]]]
- .flatMap(x => eachAppendYamlItem("", x._1, x._2, map)).toMap
+ .flatMap(x => eachAppendYamlItem("", x._1, x._2, map))
+ .toMap
} catch {
- case e: IOException => throw new IllegalArgumentException(s"Failed when loading conf error:", e)
+ case e: IOException =>
+ throw new IllegalArgumentException(s"Failed when loading conf error:", e)
}
}
@@ -99,7 +110,8 @@ object PropertiesUtils extends Logger {
properties.load(new StringReader(conf))
properties.stringPropertyNames().map(k => (k, properties.getProperty(k).trim)).toMap
} catch {
- case e: IOException => throw new IllegalArgumentException(s"Failed when loading properties ", e)
+ case e: IOException =>
+ throw new IllegalArgumentException(s"Failed when loading properties ", e)
}
}
@@ -123,22 +135,28 @@ object PropertiesUtils extends Logger {
def fromPropertiesFile(filename: String): Map[String, String] = {
val file = new File(filename)
require(file.exists(), s"[StreamPark] fromPropertiesFile: Properties file $file does not exist")
- require(file.isFile, s"[StreamPark] fromPropertiesFile: Properties file $file is not a normal file")
+ require(
+ file.isFile,
+ s"[StreamPark] fromPropertiesFile: Properties file $file is not a normal file")
val inputStream = new FileInputStream(file)
fromPropertiesFile(inputStream)
}
/** Load Yaml present in the given file. */
def fromYamlFile(inputStream: InputStream): Map[String, String] = {
- require(inputStream != null, s"[StreamPark] fromYamlFile: Properties inputStream must not be null")
+ require(
+ inputStream != null,
+ s"[StreamPark] fromYamlFile: Properties inputStream must not be null")
try {
val map = MutableMap[String, String]()
new Yaml()
.load(inputStream)
.asInstanceOf[java.util.Map[String, Map[String, Any]]]
- .flatMap(x => eachAppendYamlItem("", x._1, x._2, map)).toMap
+ .flatMap(x => eachAppendYamlItem("", x._1, x._2, map))
+ .toMap
} catch {
- case e: IOException => throw new IllegalArgumentException(s"Failed when loading yaml from inputStream", e)
+ case e: IOException =>
+ throw new IllegalArgumentException(s"Failed when loading yaml from inputStream", e)
} finally {
inputStream.close()
}
@@ -155,13 +173,16 @@ object PropertiesUtils extends Logger {
private[this] def parseHoconByReader(reader: Reader): Map[String, String] = {
try {
- ConfigFactory.parseReader(reader)
+ ConfigFactory
+ .parseReader(reader)
.entrySet()
- .map { x =>
- val k = x.getKey.trim.replaceAll("\"", "")
- val v = x.getValue.unwrapped().toString.trim
- k -> v
- }.toMap
+ .map {
+ x =>
+ val k = x.getKey.trim.replaceAll("\"", "")
+ val v = x.getValue.unwrapped().toString.trim
+ k -> v
+ }
+ .toMap
} catch {
case e: IOException => throw new IllegalArgumentException(s"Failed when loading Hocon ", e)
}
@@ -169,40 +190,56 @@ object PropertiesUtils extends Logger {
/** Load properties present in the given file. */
def fromPropertiesFile(inputStream: InputStream): Map[String, String] = {
- require(inputStream != null, s"[StreamPark] fromPropertiesFile: Properties inputStream must not be null")
+ require(
+ inputStream != null,
+ s"[StreamPark] fromPropertiesFile: Properties inputStream must not be null")
try {
val properties = new Properties()
properties.load(inputStream)
properties.stringPropertyNames().map(k => (k, properties.getProperty(k).trim)).toMap
} catch {
- case e: IOException => throw new IllegalArgumentException(s"[StreamPark] Failed when loading properties from inputStream", e)
+ case e: IOException =>
+ throw new IllegalArgumentException(
+ s"[StreamPark] Failed when loading properties from inputStream",
+ e)
}
}
- def fromYamlTextAsJava(text: String): JavaMap[String, String] = new JavaMap[String, String](fromYamlText(text).asJava)
+ def fromYamlTextAsJava(text: String): JavaMap[String, String] =
+ new JavaMap[String, String](fromYamlText(text).asJava)
- def fromHoconTextAsJava(text: String): JavaMap[String, String] = new JavaMap[String, String](fromHoconText(text).asJava)
+ def fromHoconTextAsJava(text: String): JavaMap[String, String] =
+ new JavaMap[String, String](fromHoconText(text).asJava)
- def fromPropertiesTextAsJava(text: String): JavaMap[String, String] = new JavaMap[String, String](fromPropertiesText(text).asJava)
+ def fromPropertiesTextAsJava(text: String): JavaMap[String, String] =
+ new JavaMap[String, String](fromPropertiesText(text).asJava)
- def fromYamlFileAsJava(filename: String): JavaMap[String, String] = new JavaMap[String, String](fromYamlFile(filename).asJava)
+ def fromYamlFileAsJava(filename: String): JavaMap[String, String] =
+ new JavaMap[String, String](fromYamlFile(filename).asJava)
- def fromHoconFileAsJava(filename: String): JavaMap[String, String] = new JavaMap[String, String](fromHoconFile(filename).asJava)
+ def fromHoconFileAsJava(filename: String): JavaMap[String, String] =
+ new JavaMap[String, String](fromHoconFile(filename).asJava)
- def fromPropertiesFileAsJava(filename: String): JavaMap[String, String] = new JavaMap[String, String](fromPropertiesFile(filename).asJava)
+ def fromPropertiesFileAsJava(filename: String): JavaMap[String, String] =
+ new JavaMap[String, String](fromPropertiesFile(filename).asJava)
- def fromYamlFileAsJava(inputStream: InputStream): JavaMap[String, String] = new JavaMap[String, String](fromYamlFile(inputStream).asJava)
+ def fromYamlFileAsJava(inputStream: InputStream): JavaMap[String, String] =
+ new JavaMap[String, String](fromYamlFile(inputStream).asJava)
- def fromHoconFileAsJava(inputStream: InputStream): JavaMap[String, String] = new JavaMap[String, String](fromHoconFile(inputStream).asJava)
+ def fromHoconFileAsJava(inputStream: InputStream): JavaMap[String, String] =
+ new JavaMap[String, String](fromHoconFile(inputStream).asJava)
- def fromPropertiesFileAsJava(inputStream: InputStream): JavaMap[String, String] = new JavaMap[String, String](fromPropertiesFile(inputStream).asJava)
+ def fromPropertiesFileAsJava(inputStream: InputStream): JavaMap[String, String] =
+ new JavaMap[String, String](fromPropertiesFile(inputStream).asJava)
/**
* @param file
* @return
*/
def loadFlinkConfYaml(file: File): JavaMap[String, String] = {
- require(file != null && file.exists() && file.isFile, "[StreamPark] loadFlinkConfYaml: file must not be null")
+ require(
+ file != null && file.exists() && file.isFile,
+ "[StreamPark] loadFlinkConfYaml: file must not be null")
loadFlinkConfYaml(org.apache.commons.io.FileUtils.readFileToString(file))
}
@@ -239,23 +276,23 @@ object PropertiesUtils extends Logger {
flinkConf
}
- /**
- * extract flink configuration from application.properties
- */
+ /** extract flink configuration from application.properties */
@Nonnull def extractDynamicProperties(properties: String): Map[String, String] = {
- if (StringUtils.isEmpty(properties)) Map.empty[String, String] else {
+ if (StringUtils.isEmpty(properties)) Map.empty[String, String]
+ else {
val map = mutable.Map[String, String]()
val simple = properties.replaceAll(MULTI_PROPERTY_REGEXP, "")
simple.split("\\s?-D") match {
case d if Utils.notEmpty(d) =>
- d.foreach(x => {
- if (x.nonEmpty) {
- val p = PROPERTY_PATTERN.matcher(x.trim)
- if (p.matches) {
- map += p.group(1).trim -> p.group(2).trim
+ d.foreach(
+ x => {
+ if (x.nonEmpty) {
+ val p = PROPERTY_PATTERN.matcher(x.trim)
+ if (p.matches) {
+ map += p.group(1).trim -> p.group(2).trim
+ }
}
- }
- })
+ })
case _ =>
}
val matcher = MULTI_PROPERTY_PATTERN.matcher(properties)
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/RedisClient.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/RedisClient.scala
index 3fceed985..89dbd6063 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/RedisClient.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/RedisClient.scala
@@ -17,6 +17,11 @@
package org.apache.streampark.common.util
+import org.apache.streampark.common.conf.ConfigConst
+
+import redis.clients.jedis._
+import redis.clients.jedis.exceptions.JedisConnectionException
+
import java.util.concurrent.ConcurrentHashMap
import scala.annotation.meta.getter
@@ -24,20 +29,17 @@ import scala.annotation.tailrec
import scala.collection.JavaConversions._
import scala.util.Random
-import redis.clients.jedis._
-import redis.clients.jedis.exceptions.JedisConnectionException
-
-import org.apache.streampark.common.conf.ConfigConst
-
object RedisClient extends Logger {
@transient
@getter
- private lazy val pools: ConcurrentHashMap[RedisEndpoint, JedisPool] = new ConcurrentHashMap[RedisEndpoint, JedisPool]()
+ private lazy val pools: ConcurrentHashMap[RedisEndpoint, JedisPool] =
+ new ConcurrentHashMap[RedisEndpoint, JedisPool]()
@transient
@getter
- private lazy val clusters: ConcurrentHashMap[RedisEndpoint, JedisCluster] = new ConcurrentHashMap[RedisEndpoint, JedisCluster]()
+ private lazy val clusters: ConcurrentHashMap[RedisEndpoint, JedisCluster] =
+ new ConcurrentHashMap[RedisEndpoint, JedisCluster]()
/**
* Select a random RedisEndpoint to create or get a Redis connection pool
@@ -72,7 +74,8 @@ object RedisClient extends Logger {
try {
conn = pool.getResource
} catch {
- case e: JedisConnectionException if e.getCause.toString.contains("ERR max number of clients reached") => {
+ case e: JedisConnectionException
+ if e.getCause.toString.contains("ERR max number of clients reached") => {
if (sleepTime < 500) sleepTime *= 2
Thread.sleep(sleepTime)
}
@@ -91,7 +94,13 @@ object RedisClient extends Logger {
def createJedisPool(endpoint: RedisEndpoint): JedisPool = {
val endpointEn: RedisEndpoint = endpoint.copy(auth = ConfigConst.DEFAULT_DATAMASK_STRING)
logInfo(s"[StreamPark] RedisClient: createJedisPool with $endpointEn ")
- new JedisPool(poolConfig, endpoint.host, endpoint.port, endpoint.timeout, endpoint.auth, endpoint.db)
+ new JedisPool(
+ poolConfig,
+ endpoint.host,
+ endpoint.port,
+ endpoint.timeout,
+ endpoint.auth,
+ endpoint.db)
}
private lazy val poolConfig = {
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/RedisEndpoint.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/RedisEndpoint.scala
index 188b49076..b9758beaf 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/RedisEndpoint.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/RedisEndpoint.scala
@@ -17,24 +17,28 @@
package org.apache.streampark.common.util
-import java.net.URI
-import java.util.Properties
-
-import scala.collection.JavaConversions._
+import org.apache.streampark.common.conf.ConfigConst._
import redis.clients.jedis.{Jedis, Protocol}
import redis.clients.jedis.util.{JedisClusterCRC16, JedisURIHelper, SafeEncoder}
-import org.apache.streampark.common.conf.ConfigConst._
+import java.net.URI
+import java.util.Properties
+
+import scala.collection.JavaConversions._
/**
- * RedisEndpoint represents a redis connection endpoint info: host, port, auth password
- * db number, and timeout
+ * RedisEndpoint represents a redis connection endpoint info: host, port, auth password db number,
+ * and timeout
*
- * @param host the redis host or ip
- * @param port the redis port
- * @param auth the authentication password
- * @param db database number (should be avoided in general)
+ * @param host
+ * the redis host or ip
+ * @param port
+ * the redis port
+ * @param auth
+ * the authentication password
+ * @param db
+ * database number (should be avoided in general)
*/
case class RedisEndpoint(
host: String = Protocol.DEFAULT_HOST,
@@ -42,12 +46,14 @@ case class RedisEndpoint(
auth: String = null,
db: Int = Protocol.DEFAULT_DATABASE,
timeout: Int = Protocol.DEFAULT_TIMEOUT)
- extends Serializable {
+ extends Serializable {
/**
- * Constructor from Properties. set params with redis.host, redis.port, redis.password and redis.db
+ * Constructor from Properties. set params with redis.host, redis.port, redis.password and
+ * redis.db
*
- * @param conf Properties
+ * @param conf
+ * Properties
*/
def this(conf: Properties) {
this(
@@ -61,22 +67,26 @@ case class RedisEndpoint(
/**
* Constructor with Jedis URI
*
- * @param uri connection URI in the form of redis://:$password@$host:$port/[dbnum]
+ * @param uri
+ * connection URI in the form of redis://:$password@$host:$port/[dbnum]
*/
- def this(uri: URI) = this(uri.getHost, uri.getPort, JedisURIHelper.getPassword(uri), JedisURIHelper.getDBIndex(uri))
+ def this(uri: URI) =
+ this(uri.getHost, uri.getPort, JedisURIHelper.getPassword(uri), JedisURIHelper.getDBIndex(uri))
/**
* Constructor with Jedis URI from String
*
- * @param uri connection URI in the form of redis://:$password@$host:$port/[dbnum]
+ * @param uri
+ * connection URI in the form of redis://:$password@$host:$port/[dbnum]
*/
def this(uri: String) = this(URI.create(uri))
/**
- * Connect tries to open a connection to the redis endpoint,
- * optionally authenticating and selecting a db
+ * Connect tries to open a connection to the redis endpoint, optionally authenticating and
+ * selecting a db
*
- * @return a new Jedis instance
+ * @return
+ * a new Jedis instance
*/
def connect(): Jedis = RedisClient.connect(this)
@@ -89,8 +99,8 @@ case class RedisNode(endpoint: RedisEndpoint, startSlot: Int, endSlot: Int, idx:
}
/**
- * RedisConfig holds the state of the cluster nodes, and uses consistent hashing to map
- * keys to nodes
+ * RedisConfig holds the state of the cluster nodes, and uses consistent hashing to map keys to
+ * nodes
*/
class RedisConfig(val initialHost: RedisEndpoint) extends Serializable {
@@ -99,14 +109,10 @@ class RedisConfig(val initialHost: RedisEndpoint) extends Serializable {
val hosts = getHosts(initialHost)
val nodes = getNodes(initialHost)
- /**
- * @return initialHost's auth
- */
+ /** @return initialHost's auth */
def getAuth: String = initialHost.auth
- /**
- * @return selected db number
- */
+ /** @return selected db number */
def getDB: Int = initialHost.db
def getRandomNode: RedisNode = {
@@ -115,30 +121,38 @@ class RedisConfig(val initialHost: RedisEndpoint) extends Serializable {
}
/**
- * @param sPos start slot number
- * @param ePos end slot number
- * @return a list of RedisNode whose slots union [sPos, ePos] is not null
+ * @param sPos
+ * start slot number
+ * @param ePos
+ * end slot number
+ * @return
+ * a list of RedisNode whose slots union [sPos, ePos] is not null
*/
def getNodesBySlots(sPos: Int, ePos: Int): Array[RedisNode] = {
/* This function judges if [sPos1, ePos1] union [sPos2, ePos2] is not null */
def inter(sPos1: Int, ePos1: Int, sPos2: Int, ePos2: Int) =
if (sPos1 <= sPos2) ePos1 >= sPos2 else ePos2 >= sPos1
- nodes.filter(node => inter(sPos, ePos, node.startSlot, node.endSlot)).filter(_.idx == 0) // master only now
+ nodes
+ .filter(node => inter(sPos, ePos, node.startSlot, node.endSlot))
+ .filter(_.idx == 0) // master only now
}
/**
* @param key
- * *IMPORTANT* Please remember to close after using
- * @return jedis who is a connection for a given key
+ * *IMPORTANT* Please remember to close after using
+ * @return
+ * jedis who is a connection for a given key
*/
def connectionForKey(key: String): Jedis = {
getHost(key).connect()
}
/**
- * @param initialHost any redis endpoint of a cluster or a single server
- * @return true if the target server is in cluster mode
+ * @param initialHost
+ * any redis endpoint of a cluster or a single server
+ * @return
+ * true if the target server is in cluster mode
*/
private def clusterEnabled(initialHost: RedisEndpoint): Boolean = {
val conn = initialHost.connect()
@@ -153,26 +167,32 @@ class RedisConfig(val initialHost: RedisEndpoint) extends Serializable {
/**
* @param key
- * @return host whose slots should involve key
+ * @return
+ * host whose slots should involve key
*/
def getHost(key: String): RedisNode = {
val slot = JedisClusterCRC16.getSlot(key)
- hosts.filter(host => {
- host.startSlot <= slot && host.endSlot >= slot
- })(0)
+ hosts.filter(
+ host => {
+ host.startSlot <= slot && host.endSlot >= slot
+ })(0)
}
/**
- * @param initialHost any redis endpoint of a cluster or a single server
- * @return list of host nodes
+ * @param initialHost
+ * any redis endpoint of a cluster or a single server
+ * @return
+ * list of host nodes
*/
private def getHosts(initialHost: RedisEndpoint): Array[RedisNode] = {
getNodes(initialHost).filter(_.idx == 0)
}
/**
- * @param initialHost any redis endpoint of a single server
- * @return list of nodes
+ * @param initialHost
+ * any redis endpoint of a single server
+ * @return
+ * list of nodes
*/
private def getNonClusterNodes(initialHost: RedisEndpoint): Array[RedisNode] = {
val master = (initialHost.host, initialHost.port)
@@ -192,56 +212,85 @@ class RedisConfig(val initialHost: RedisEndpoint) extends Serializable {
} else {
// this is a master - take its slaves
- val slaves = replinfo.filter(x => x.contains("slave") && x.contains("online")).map(rl => {
- val content = rl.substring(rl.indexOf(':') + 1).split(",")
- val ip = content(0)
- val port = content(1)
- (ip.substring(ip.indexOf('=') + 1), port.substring(port.indexOf('=') + 1).toInt)
- })
+ val slaves = replinfo
+ .filter(x => x.contains("slave") && x.contains("online"))
+ .map(
+ rl => {
+ val content = rl.substring(rl.indexOf(':') + 1).split(",")
+ val ip = content(0)
+ val port = content(1)
+ (ip.substring(ip.indexOf('=') + 1), port.substring(port.indexOf('=') + 1).toInt)
+ })
val nodes = master +: slaves
val range = nodes.length
- (0 until range).map(i =>
- RedisNode(RedisEndpoint(nodes(i)._1, nodes(i)._2, initialHost.auth, initialHost.db, initialHost.timeout), 0, 16383, i, range)).toArray
+ (0 until range)
+ .map(
+ i =>
+ RedisNode(
+ RedisEndpoint(
+ nodes(i)._1,
+ nodes(i)._2,
+ initialHost.auth,
+ initialHost.db,
+ initialHost.timeout),
+ 0,
+ 16383,
+ i,
+ range))
+ .toArray
}
}
/**
- * @param initialHost any redis endpoint of a cluster server
- * @return list of nodes
+ * @param initialHost
+ * any redis endpoint of a cluster server
+ * @return
+ * list of nodes
*/
private def getClusterNodes(initialHost: RedisEndpoint): Array[RedisNode] = {
val conn = initialHost.connect()
- val res = conn.clusterSlots().flatMap {
- slotInfoObj =>
- {
- val slotInfo = slotInfoObj.asInstanceOf[java.util.List[java.lang.Object]]
- val sPos = slotInfo.get(0).toString.toInt
- val ePos = slotInfo.get(1).toString.toInt
- /*
- * We will get all the nodes with the slots range [sPos, ePos],
- * and create RedisNode for each nodes, the total field of all
- * RedisNode are the number of the nodes whose slots range is
- * as above, and the idx field is just an index for each node
- * which will be used for adding support for slaves and so on.
- * And the idx of a master is always 0, we rely on this fact to
- * filter master.
- */
- (0 until (slotInfo.size - 2)).map(i => {
- val node = slotInfo(i + 2).asInstanceOf[java.util.List[java.lang.Object]]
- val host = SafeEncoder.encode(node.get(0).asInstanceOf[Array[scala.Byte]])
- val port = node.get(1).toString.toInt
- RedisNode(RedisEndpoint(host, port, initialHost.auth, initialHost.db, initialHost.timeout), sPos, ePos, i, slotInfo.size - 2)
- })
- }
- }.toArray
+ val res = conn
+ .clusterSlots()
+ .flatMap {
+ slotInfoObj =>
+ {
+ val slotInfo = slotInfoObj.asInstanceOf[java.util.List[java.lang.Object]]
+ val sPos = slotInfo.get(0).toString.toInt
+ val ePos = slotInfo.get(1).toString.toInt
+ /*
+ * We will get all the nodes with the slots range [sPos, ePos],
+ * and create RedisNode for each nodes, the total field of all
+ * RedisNode are the number of the nodes whose slots range is
+ * as above, and the idx field is just an index for each node
+ * which will be used for adding support for slaves and so on.
+ * And the idx of a master is always 0, we rely on this fact to
+ * filter master.
+ */
+ (0 until (slotInfo.size - 2)).map(
+ i => {
+ val node = slotInfo(i + 2).asInstanceOf[java.util.List[java.lang.Object]]
+ val host = SafeEncoder.encode(node.get(0).asInstanceOf[Array[scala.Byte]])
+ val port = node.get(1).toString.toInt
+ RedisNode(
+ RedisEndpoint(host, port, initialHost.auth, initialHost.db, initialHost.timeout),
+ sPos,
+ ePos,
+ i,
+ slotInfo.size - 2)
+ })
+ }
+ }
+ .toArray
conn.close()
res
}
/**
- * @param initialHost any redis endpoint of a cluster or a single server
- * @return list of nodes
+ * @param initialHost
+ * any redis endpoint of a cluster or a single server
+ * @return
+ * list of nodes
*/
def getNodes(initialHost: RedisEndpoint): Array[RedisNode] = {
if (clusterEnabled(initialHost)) {
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/RedisUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/RedisUtils.scala
index b88841ed5..477f3cffb 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/RedisUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/RedisUtils.scala
@@ -16,6 +16,8 @@
*/
package org.apache.streampark.common.util
+import redis.clients.jedis.{Jedis, JedisCluster, Pipeline, ScanParams}
+
import java.lang.{Integer => JInt}
import java.util.Set
@@ -24,30 +26,32 @@ import scala.collection.JavaConverters._
import scala.collection.immutable
import scala.util.{Failure, Success, Try}
-import redis.clients.jedis.{Jedis, JedisCluster, Pipeline, ScanParams}
-
object RedisUtils extends Logger {
def exists(key: String)(implicit endpoint: RedisEndpoint): Boolean = doRedis(_.exists(key))
- def hexists(key: String, field: String, func: () => Unit = null)(implicit endpoint: RedisEndpoint): Boolean = doRedis(_.hexists(key, field), func)
+ def hexists(key: String, field: String, func: () => Unit = null)(implicit
+ endpoint: RedisEndpoint): Boolean = doRedis(_.hexists(key, field), func)
def get(key: String)(implicit endpoint: RedisEndpoint): String = doRedis(_.get(key))
- def hget(key: String, field: String)(implicit endpoint: RedisEndpoint): String = doRedis(_.hget(key, field))
+ def hget(key: String, field: String)(implicit endpoint: RedisEndpoint): String = doRedis(
+ _.hget(key, field))
/**
- * The Redis Setex command sets the value and expiration time. If the key already exists,
- * the Setex command will replace the old value.
+ * The Redis Setex command sets the value and expiration time. If the key already exists, the
+ * Setex command will replace the old value.
*/
- def setex(key: String, value: String, ttl: JInt = null, func: () => Unit = null)(implicit endpoint: RedisEndpoint): String =
+ def setex(key: String, value: String, ttl: JInt = null, func: () => Unit = null)(implicit
+ endpoint: RedisEndpoint): String =
doRedis(_.setex(key, ttl, value), func)
/**
- * The Redis Setnx (SET if Not eXists) command sets the specified value
- * if the specified key does not exist
+ * The Redis Setnx (SET if Not eXists) command sets the specified value if the specified key does
+ * not exist
*/
- def setnx(key: String, value: String, ttl: JInt = null, func: () => Unit = null)(implicit endpoint: RedisEndpoint): Long = doRedis(
+ def setnx(key: String, value: String, ttl: JInt = null, func: () => Unit = null)(implicit
+ endpoint: RedisEndpoint): Long = doRedis(
r => {
val x = r.setnx(key, value)
if (x == 1 && ttl != null) {
@@ -59,10 +63,11 @@ object RedisUtils extends Logger {
/**
* The Redis Hsetnx command is used to assign values to fields in a hash table that do not exist.
- * If the hash table does not exist, a new hash table is created and the HSET operation is performed.
- * If the field already exists in the hash table, the operation is invalid
+ * If the hash table does not exist, a new hash table is created and the HSET operation is
+ * performed. If the field already exists in the hash table, the operation is invalid
*/
- def hsetnx(key: String, field: String, value: String, ttl: JInt = null, func: () => Unit = null)(implicit endpoint: RedisEndpoint): Long = doRedis(
+ def hsetnx(key: String, field: String, value: String, ttl: JInt = null, func: () => Unit = null)(
+ implicit endpoint: RedisEndpoint): Long = doRedis(
r => {
val x = r.hsetnx(key, field, value)
if (x == 1 && ttl != null) {
@@ -72,11 +77,14 @@ object RedisUtils extends Logger {
},
func)
- def mget(keys: Array[String])(implicit endpoint: RedisEndpoint): Array[String] = doRedis(_.mget(keys: _*).asScala.toArray)
+ def mget(keys: Array[String])(implicit endpoint: RedisEndpoint): Array[String] = doRedis(
+ _.mget(keys: _*).asScala.toArray)
- def del(key: String, func: () => Unit = null)(implicit endpoint: RedisEndpoint): Long = doRedis(_.del(key), func)
+ def del(key: String, func: () => Unit = null)(implicit endpoint: RedisEndpoint): Long =
+ doRedis(_.del(key), func)
- def set(key: String, value: String, ttl: JInt = null, func: () => Unit = null)(implicit endpoint: RedisEndpoint): String = doRedis(
+ def set(key: String, value: String, ttl: JInt = null, func: () => Unit = null)(implicit
+ endpoint: RedisEndpoint): String = doRedis(
r => {
val s = r.set(key, value)
if (ttl != null) {
@@ -86,7 +94,8 @@ object RedisUtils extends Logger {
},
func)
- def hset(key: String, field: String, value: String, ttl: JInt = null, func: () => Unit = null)(implicit endpoint: RedisEndpoint): Long = doRedis(
+ def hset(key: String, field: String, value: String, ttl: JInt = null, func: () => Unit = null)(
+ implicit endpoint: RedisEndpoint): Long = doRedis(
r => {
val s = r.hset(key, field, value)
if (ttl != null) {
@@ -96,7 +105,8 @@ object RedisUtils extends Logger {
},
func)
- def hmset(key: String, hash: Map[String, String], ttl: JInt = null, func: () => Unit = null)(implicit endpoint: RedisEndpoint): String = doRedis(
+ def hmset(key: String, hash: Map[String, String], ttl: JInt = null, func: () => Unit = null)(
+ implicit endpoint: RedisEndpoint): String = doRedis(
r => {
val s = r.hmset(key, hash.asJava)
if (ttl != null) {
@@ -106,16 +116,20 @@ object RedisUtils extends Logger {
},
func)
- def hmget(key: String, fields: String*)(implicit endpoint: RedisEndpoint): List[String] = doRedis(_.hmget(key, fields: _*).toList)
+ def hmget(key: String, fields: String*)(implicit endpoint: RedisEndpoint): List[String] = doRedis(
+ _.hmget(key, fields: _*).toList)
- def hgetAll(key: String)(implicit endpoint: RedisEndpoint): Map[String, String] = doRedis(_.hgetAll(key).toMap)
+ def hgetAll(key: String)(implicit endpoint: RedisEndpoint): Map[String, String] = doRedis(
+ _.hgetAll(key).toMap)
- def hdel(key: String, fields: immutable.List[String], func: () => Unit = null)(implicit endpoint: RedisEndpoint): Long = {
+ def hdel(key: String, fields: immutable.List[String], func: () => Unit = null)(implicit
+ endpoint: RedisEndpoint): Long = {
if (key == null || fields == null || fields.isEmpty) 0L
else doRedis(_.hdel(key, fields.toArray: _*), func)
}
- def sadd(key: String, members: immutable.List[String], ttl: JInt = null, func: () => Unit = null)(implicit endpoint: RedisEndpoint): Long = {
+ def sadd(key: String, members: immutable.List[String], ttl: JInt = null, func: () => Unit = null)(
+ implicit endpoint: RedisEndpoint): Long = {
doRedis(
r => {
val res = r.sadd(key, members.toArray: _*)
@@ -127,12 +141,19 @@ object RedisUtils extends Logger {
func)
}
- def smembers(key: String)(implicit endpoint: RedisEndpoint): Set[String] = doRedis(_.smembers(key))
+ def smembers(key: String)(implicit endpoint: RedisEndpoint): Set[String] = doRedis(
+ _.smembers(key))
- def srem(key: String, members: immutable.List[String], func: () => Unit = null)(implicit endpoint: RedisEndpoint): Long =
+ def srem(key: String, members: immutable.List[String], func: () => Unit = null)(implicit
+ endpoint: RedisEndpoint): Long =
doRedis(_.srem(key, members.toArray: _*), func)
- def getOrElseHset(key: String, field: String, value: String, ttl: JInt = null, func: () => Unit = null)(implicit endpoint: RedisEndpoint): String = doRedis(
+ def getOrElseHset(
+ key: String,
+ field: String,
+ value: String,
+ ttl: JInt = null,
+ func: () => Unit = null)(implicit endpoint: RedisEndpoint): String = doRedis(
x => {
val v = x.hget(key, field)
if (v == null) {
@@ -145,7 +166,8 @@ object RedisUtils extends Logger {
},
func)
- def getOrElseSet(key: String, value: String, ttl: JInt = null, func: () => Unit = null)(implicit endpoint: RedisEndpoint): String = doRedis(
+ def getOrElseSet(key: String, value: String, ttl: JInt = null, func: () => Unit = null)(implicit
+ endpoint: RedisEndpoint): String = doRedis(
x => {
val v = x.get(key)
if (v == null) {
@@ -158,7 +180,8 @@ object RedisUtils extends Logger {
},
func)
- def hincrBy(key: String, field: String, value: Long, ttl: JInt = null, func: () => Unit = null)(implicit endpoint: RedisEndpoint): Long = doRedis(
+ def hincrBy(key: String, field: String, value: Long, ttl: JInt = null, func: () => Unit = null)(
+ implicit endpoint: RedisEndpoint): Long = doRedis(
x => {
val reply = x.hincrBy(key, field, value)
if (ttl != null) {
@@ -168,7 +191,12 @@ object RedisUtils extends Logger {
},
func)
- def hincrByFloat(key: String, field: String, value: Double, ttl: JInt = null, func: () => Unit = null)(implicit endpoint: RedisEndpoint): Double = doRedis(
+ def hincrByFloat(
+ key: String,
+ field: String,
+ value: Double,
+ ttl: JInt = null,
+ func: () => Unit = null)(implicit endpoint: RedisEndpoint): Double = doRedis(
x => {
val reply = x.hincrByFloat(key, field, value)
if (ttl != null) {
@@ -178,7 +206,8 @@ object RedisUtils extends Logger {
},
func)
- def incrBy(key: String, value: Long, ttl: JInt = null, func: () => Unit = null)(implicit endpoint: RedisEndpoint): Long = doRedis(
+ def incrBy(key: String, value: Long, ttl: JInt = null, func: () => Unit = null)(implicit
+ endpoint: RedisEndpoint): Long = doRedis(
x => {
val reply = x.incrBy(key, value)
if (ttl != null) {
@@ -188,7 +217,8 @@ object RedisUtils extends Logger {
},
func)
- def incrByFloat(key: String, value: Double, ttl: JInt = null, func: () => Unit = null)(implicit endpoint: RedisEndpoint): Double = doRedis(
+ def incrByFloat(key: String, value: Double, ttl: JInt = null, func: () => Unit = null)(implicit
+ endpoint: RedisEndpoint): Double = doRedis(
x => {
val reply = x.incrByFloat(key, value)
if (ttl != null) {
@@ -198,58 +228,65 @@ object RedisUtils extends Logger {
},
func)
- /**
- * Batch writing
- */
- def mSets(kvs: Seq[(String, String)], ttl: JInt = null, func: () => Unit = null)(implicit endpoint: RedisEndpoint): Long = doRedis(
+ /** Batch writing */
+ def mSets(kvs: Seq[(String, String)], ttl: JInt = null, func: () => Unit = null)(implicit
+ endpoint: RedisEndpoint): Long = doRedis(
x => {
val start = System.currentTimeMillis()
val pipe = x.pipelined()
- kvs.foreach { case (k, v) =>
- pipe.mset(k, v)
- if (ttl != null) {
- pipe.expire(k, ttl)
- }
+ kvs.foreach {
+ case (k, v) =>
+ pipe.mset(k, v)
+ if (ttl != null) {
+ pipe.expire(k, ttl)
+ }
}
pipe.sync()
System.currentTimeMillis() - start
},
- func)
+ func
+ )
- def mSetex(kvs: Seq[(String, String)], ttl: JInt = null, func: () => Unit = null)(implicit endpoint: RedisEndpoint): Long = doRedis(
+ def mSetex(kvs: Seq[(String, String)], ttl: JInt = null, func: () => Unit = null)(implicit
+ endpoint: RedisEndpoint): Long = doRedis(
x => {
val start = System.currentTimeMillis()
val pipe = x.pipelined()
- kvs.foreach { case (k, v) =>
- pipe.setnx(k, v)
- if (ttl != null) {
- pipe.expire(k, ttl)
- }
+ kvs.foreach {
+ case (k, v) =>
+ pipe.setnx(k, v)
+ if (ttl != null) {
+ pipe.expire(k, ttl)
+ }
}
pipe.sync()
System.currentTimeMillis() - start
},
- func)
+ func
+ )
- def expire(key: String, s: Int)(implicit endpoint: RedisEndpoint): Long = doRedis(_.expire(key, s))
+ def expire(key: String, s: Int)(implicit endpoint: RedisEndpoint): Long = doRedis(
+ _.expire(key, s))
- def delByPattern(key: String, func: () => Unit = null)(implicit endpoint: RedisEndpoint): Any = doRedis(
- r => {
- // Use scan to delete 10,000 records at a time, to prevent a load of too many records, caused oom
- val scanParams = new ScanParams
- scanParams.`match`(key)
- scanParams.count(10000)
- var cursor = ScanParams.SCAN_POINTER_START
- do {
- val scanResult = r.scan(cursor, scanParams)
- cursor = scanResult.getCursor
- val keys = scanResult.getResult.toList
- if (keys.nonEmpty) {
- r.del(keys: _*)
- }
- } while (cursor != "0")
- },
- func)
+ def delByPattern(key: String, func: () => Unit = null)(implicit endpoint: RedisEndpoint): Any =
+ doRedis(
+ r => {
+ // Use scan to delete 10,000 records at a time, to prevent a load of too many records, caused oom
+ val scanParams = new ScanParams
+ scanParams.`match`(key)
+ scanParams.count(10000)
+ var cursor = ScanParams.SCAN_POINTER_START
+ do {
+ val scanResult = r.scan(cursor, scanParams)
+ cursor = scanResult.getCursor
+ val keys = scanResult.getResult.toList
+ if (keys.nonEmpty) {
+ r.del(keys: _*)
+ }
+ } while (cursor != "0")
+ },
+ func
+ )
def hlen(key: String)(implicit endpoint: RedisEndpoint): Long = doRedis(_.hlen(key))
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/ReflectUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/ReflectUtils.scala
index 0ec30f97d..092bf3449 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/ReflectUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/ReflectUtils.scala
@@ -17,26 +17,31 @@
package org.apache.streampark.common.util
+import org.apache.commons.lang3.StringUtils
+
import java.lang.reflect.{Field, Modifier}
import java.util.Objects
import scala.util.{Failure, Success, Try}
-import org.apache.commons.lang3.StringUtils
-
object ReflectUtils extends Logger {
/**
* Get field.
*
- * @param beanClass the bean class
- * @param name the name
- * @return the field
- * @throws SecurityException the security exception
+ * @param beanClass
+ * the bean class
+ * @param name
+ * the name
+ * @return
+ * the field
+ * @throws SecurityException
+ * the security exception
*/
@throws[SecurityException]
def getField(beanClass: Class[_], name: String): Field = {
- Try(beanClass.getDeclaredFields.filter(f => Objects.equals(name, f.getName)).head).getOrElse(null)
+ Try(beanClass.getDeclaredFields.filter(f => Objects.equals(name, f.getName)).head)
+ .getOrElse(null)
}
def getFieldValue(obj: Any, fieldName: String): Any = {
@@ -57,7 +62,9 @@ object ReflectUtils extends Logger {
def setFieldValue(obj: Any, fieldName: String, value: Any): Unit = {
val field = getAccessibleField(obj, fieldName)
- if (Objects.isNull(field)) throw new IllegalArgumentException("Could not find field [" + fieldName + "] on target [" + obj + "]")
+ if (Objects.isNull(field))
+ throw new IllegalArgumentException(
+ "Could not find field [" + fieldName + "] on target [" + obj + "]")
try
field.set(obj, value)
catch {
@@ -85,9 +92,11 @@ object ReflectUtils extends Logger {
}
private def makeAccessible(field: Field): Unit = {
- if ((!Modifier.isPublic(field.getModifiers)
+ if (
+ (!Modifier.isPublic(field.getModifiers)
|| !Modifier.isPublic(field.getDeclaringClass.getModifiers)
- || Modifier.isFinal(field.getModifiers)) && !field.isAccessible) {
+ || Modifier.isFinal(field.getModifiers)) && !field.isAccessible
+ ) {
field.setAccessible(true)
}
}
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/SqlConvertUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/SqlConvertUtils.scala
index 90def5fa0..0c315ac98 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/SqlConvertUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/SqlConvertUtils.scala
@@ -29,9 +29,8 @@ object SqlConvertUtils extends Logger {
"\\s*(.*?)\\s+(([a-z]+)\\((.*?)\\)|[a-z]+)(\\s*|((.*?)(comment)\\s+(['|\"](.*?)['|\"])|(.*?))),$",
Pattern.CASE_INSENSITIVE)
- private[this] val PRIMARY_REGEXP = Pattern.compile(
- "primary\\s+key\\s+\\((.*?)\\)",
- Pattern.CASE_INSENSITIVE)
+ private[this] val PRIMARY_REGEXP =
+ Pattern.compile("primary\\s+key\\s+\\((.*?)\\)", Pattern.CASE_INSENSITIVE)
/**
* convert to flink types
@@ -68,7 +67,8 @@ object SqlConvertUtils extends Logger {
case "BIGINT" | "INT" =>
length match {
case null => "Int32"
- case x => x.trim.toInt match {
+ case x =>
+ x.trim.toInt match {
case x if x < 3 => "Int8"
case x if x < 5 => "Int16"
case x if x < 9 => "Int32"
@@ -101,7 +101,10 @@ object SqlConvertUtils extends Logger {
val COMMENT_REGEXP = Pattern.compile("(comment)\\s+(['\"])", Pattern.CASE_INSENSITIVE)
- @tailrec def commentJoin(map: util.Map[Integer, String], index: Integer, segment: String): (Integer, String) = {
+ @tailrec def commentJoin(
+ map: util.Map[Integer, String],
+ index: Integer,
+ segment: String): (Integer, String) = {
val matcher = COMMENT_REGEXP.matcher(segment)
matcher.find() match {
case b if !b => index -> segment
@@ -132,7 +135,8 @@ object SqlConvertUtils extends Logger {
}
}
- val body = sql.substring(sql.indexOf("("), sql.lastIndexOf(")") + 1)
+ val body = sql
+ .substring(sql.indexOf("("), sql.lastIndexOf(")") + 1)
.replaceAll("\r|\n|\r\n", "")
.replaceFirst("\\(", "(\n")
.replaceFirst("\\)$", "\n)")
@@ -145,36 +149,45 @@ object SqlConvertUtils extends Logger {
}
val sqlBuffer = new StringBuffer(sql.substring(0, sql.indexOf("(")))
var skipNo: Int = -1
- map.foreach(a => {
- if (a._1 > skipNo) {
- val length = lengthJoin(map, a._1, a._2)
- if (length._1 > a._1) {
- sqlBuffer.append(length._2).append("\n")
- skipNo = length._1
- } else {
- val comment = commentJoin(map, a._1, a._2)
- if (comment._1 > a._1) {
- sqlBuffer.append(comment._2).append("\n")
- skipNo = comment._1
+ map.foreach(
+ a => {
+ if (a._1 > skipNo) {
+ val length = lengthJoin(map, a._1, a._2)
+ if (length._1 > a._1) {
+ sqlBuffer.append(length._2).append("\n")
+ skipNo = length._1
} else {
- sqlBuffer.append(a._2).append("\n")
+ val comment = commentJoin(map, a._1, a._2)
+ if (comment._1 > a._1) {
+ sqlBuffer.append(comment._2).append("\n")
+ skipNo = comment._1
+ } else {
+ sqlBuffer.append(a._2).append("\n")
+ }
}
}
- }
- })
+ })
scanner.close()
sqlBuffer.toString.trim.concat(sql.substring(sql.lastIndexOf(")") + 1))
}
/**
- * @param sql : original sql statement to be converted
- * @param typeFunc : type conversion function
- * @param keyFunc : handle primary key function
- * @param postfix : suffix content
+ * @param sql
+ * : original sql statement to be converted
+ * @param typeFunc
+ * : type conversion function
+ * @param keyFunc
+ * : handle primary key function
+ * @param postfix
+ * : suffix content
* @return
*/
- private[this] def convertSql(sql: String, typeFunc: (String, String) => String = null, keyFunc: String => String = null, postfix: String = null): String = {
+ private[this] def convertSql(
+ sql: String,
+ typeFunc: (String, String) => String = null,
+ keyFunc: String => String = null,
+ postfix: String = null): String = {
val formattedSql = formatSql(sql)
val scanner = new Scanner(formattedSql)
@@ -212,7 +225,9 @@ object SqlConvertUtils extends Logger {
}
}
if (line != null) {
- sqlBuffer.append(line).append(if (line.toUpperCase.trim.startsWith("CREATE ")) "\n" else ",\n")
+ sqlBuffer
+ .append(line)
+ .append(if (line.toUpperCase.trim.startsWith("CREATE ")) "\n" else ",\n")
}
}
scanner.close()
@@ -234,6 +249,7 @@ object SqlConvertUtils extends Logger {
},
postfix)
- def mysqlToClickhouse(sql: String, postfix: String): String = convertSql(sql, toClickhouseDataType, postfix = postfix)
+ def mysqlToClickhouse(sql: String, postfix: String): String =
+ convertSql(sql, toClickhouseDataType, postfix = postfix)
}
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/SystemPropertyUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/SystemPropertyUtils.scala
index e784fd293..11dad97ff 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/SystemPropertyUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/SystemPropertyUtils.scala
@@ -24,16 +24,16 @@ import scala.util.{Failure, Success, Try}
object SystemPropertyUtils extends Logger {
/**
- * Returns {@code true} if and only if the system property with the specified {@code key}
- * exists.
+ * Returns {@code true} if and only if the system property with the specified {@code key} exists.
*/
def contains(key: String): Boolean = get(key) != null
/**
- * Returns the value of the Java system property with the specified
- * {@code key}, while falling back to {@code null} if the property access fails.
+ * Returns the value of the Java system property with the specified {@code key}, while falling
+ * back to {@code null} if the property access fails.
*
- * @return the property value or { @code null}
+ * @return
+ * the property value or { @code null}
*/
def get(key: String): String = get(key, null)
@@ -45,7 +45,8 @@ object SystemPropertyUtils extends Logger {
Try {
System.getSecurityManager match {
case null => System.getProperty(other)
- case _ => AccessController.doPrivileged(new PrivilegedAction[String]() {
+ case _ =>
+ AccessController.doPrivileged(new PrivilegedAction[String]() {
override def run: String = System.getProperty(other)
})
}
@@ -56,7 +57,8 @@ object SystemPropertyUtils extends Logger {
case value => value
}
case Failure(e) =>
- logger.warn(s"Unable to retrieve a system property '$other'; default values will be used, ${e.getMessage}.")
+ logger.warn(
+ s"Unable to retrieve a system property '$other'; default values will be used, ${e.getMessage}.")
default
}
}
@@ -70,31 +72,29 @@ object SystemPropertyUtils extends Logger {
case "false" | "no" | "0" => false
case other: String if other.isEmpty => false
case _ =>
- logger.warn(s"Unable to parse the boolean system property '$key':$value - using the default value: $default.")
+ logger.warn(
+ s"Unable to parse the boolean system property '$key':$value - using the default value: $default.")
default
}
}
def getInt(key: String, default: Int): Int = {
- Try(
- get(key).toInt) match {
+ Try(get(key).toInt) match {
case Success(ok) => ok
case Failure(_) => default
}
}
def getLong(key: String, default: Long): Long = {
- Try(
- get(key).toLong) match {
+ Try(get(key).toLong) match {
case Success(ok) => ok
case Failure(_) => default
}
}
- /**
- * Sets the value of the Java system property with the specified {@code key}
- */
- def set(key: String, value: String): String = System.getProperties.setProperty(key, value).asInstanceOf[String]
+ /** Sets the value of the Java system property with the specified {@code key} */
+ def set(key: String, value: String): String =
+ System.getProperties.setProperty(key, value).asInstanceOf[String]
def getOrElseUpdate(key: String, default: String): String = {
get(key) match {
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/ThreadUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/ThreadUtils.scala
index 0d888ce49..4f35f5465 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/ThreadUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/ThreadUtils.scala
@@ -16,10 +16,10 @@
*/
package org.apache.streampark.common.util
-import java.util.concurrent.{ExecutorService, ThreadFactory, TimeUnit}
-
import com.google.common.util.concurrent.ThreadFactoryBuilder
+import java.util.concurrent.{ExecutorService, ThreadFactory, TimeUnit}
+
object ThreadUtils {
def threadFactory(threadName: String, isDaemon: Boolean): ThreadFactory =
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/Utils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/Utils.scala
index b89eec0a8..71e807062 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/Utils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/Utils.scala
@@ -16,29 +16,16 @@
*/
package org.apache.streampark.common.util
-import java.io.{
- BufferedInputStream,
- File,
- FileInputStream,
- IOException,
- PrintWriter,
- StringWriter
-}
-import java.lang.{
- Boolean => JavaBool,
- Double => JavaDouble,
- Float => JavaFloat,
- Integer => JavaInt,
- Long => JavaLong,
- Short => JavaShort,
- Byte => JavaByte
-}
+import org.apache.commons.lang3.StringUtils
+
+import java.io.{BufferedInputStream, File, FileInputStream, IOException, PrintWriter, StringWriter}
+import java.lang.{Boolean => JavaBool, Byte => JavaByte, Double => JavaDouble, Float => JavaFloat, Integer => JavaInt, Long => JavaLong, Short => JavaShort}
import java.net.URL
-import java.util.{Properties, UUID, jar, Collection => JavaCollection, Map => JavaMap}
+import java.util.{jar, Collection => JavaCollection, Map => JavaMap, Properties, UUID}
import java.util.jar.{JarFile, JarInputStream}
+
import scala.collection.JavaConversions._
import scala.util.{Failure, Success, Try}
-import org.apache.commons.lang3.StringUtils
object Utils {
@@ -96,7 +83,8 @@ object Utils {
throw new IOException(s"JAR file can't be read '${jarFile.getAbsolutePath}'")
}
Try(new JarFile(jarFile)) match {
- case Failure(e) => throw new IOException(s"Error while opening jar file '${jarFile.getAbsolutePath}'", e)
+ case Failure(e) =>
+ throw new IOException(s"Error while opening jar file '${jarFile.getAbsolutePath}'", e)
case Success(x) => x.close()
}
}
@@ -106,26 +94,24 @@ object Utils {
new JarInputStream(new BufferedInputStream(new FileInputStream(jarFile))).getManifest
}
- def copyProperties(original: Properties, target: Properties): Unit = original.foreach(x => target.put(x._1, x._2))
+ def copyProperties(original: Properties, target: Properties): Unit =
+ original.foreach(x => target.put(x._1, x._2))
- /**
- * get os name
- */
+ /** get os name */
def getOsName: String = OS
def isLinux: Boolean = OS.indexOf("linux") >= 0
def isWindows: Boolean = OS.indexOf("windows") >= 0
- /**
- * if any blank strings exist
- */
+ /** if any blank strings exist */
def isAnyBank(items: String*): Boolean = items == null || items.exists(StringUtils.isBlank)
/*
* Mimicking the try-with-resource syntax of Java-8+
*/
- def tryWithResource[R, T <: AutoCloseable](handle: T)(func: T => R)(implicit excFunc: Throwable => R = null): R = {
+ def tryWithResource[R, T <: AutoCloseable](handle: T)(func: T => R)(implicit
+ excFunc: Throwable => R = null): R = {
try {
func(handle)
} catch {
@@ -138,19 +124,21 @@ object Utils {
}
def close(closeable: AutoCloseable*)(implicit func: Throwable => Unit = null): Unit = {
- closeable.foreach(c => {
- try {
- if (c != null) {
- c.close()
+ closeable.foreach(
+ c => {
+ try {
+ if (c != null) {
+ c.close()
+ }
+ } catch {
+ case e: Throwable if func != null => func(e)
}
- } catch {
- case e: Throwable if func != null => func(e)
- }
- })
+ })
}
/**
- * calculate the percentage of num1 / num2, the result range from 0 to 100, with one small digit reserve.
+ * calculate the percentage of num1 / num2, the result range from 0 to 100, with one small digit
+ * reserve.
*/
def calPercent(num1: Long, num2: Long): Double =
if (num1 == 0 || num2 == 0) 0.0
@@ -167,7 +155,8 @@ object Utils {
}
def stringifyException(e: Throwable): String = {
- if (e == null) "(null)" else {
+ if (e == null) "(null)"
+ else {
try {
val stm = new StringWriter
val wrt = new PrintWriter(stm)
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/YarnUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/YarnUtils.scala
index 84d01d3be..d9058de3d 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/YarnUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/YarnUtils.scala
@@ -16,15 +16,7 @@
*/
package org.apache.streampark.common.util
-import java.net.InetAddress
-import java.security.PrivilegedExceptionAction
-import java.util
-import java.util.{HashMap => JavaHashMap, List => JavaList}
-
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
-import scala.util.{Failure, Success, Try}
-import scala.util.control.Breaks.{break, breakable}
+import org.apache.streampark.common.conf.{CommonConfig, InternalConfigHolder}
import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.fs.CommonConfigurationKeys
@@ -38,7 +30,15 @@ import org.apache.http.client.methods.HttpGet
import org.apache.http.client.protocol.HttpClientContext
import org.apache.http.impl.client.HttpClients
-import org.apache.streampark.common.conf.{CommonConfig, InternalConfigHolder}
+import java.net.InetAddress
+import java.security.PrivilegedExceptionAction
+import java.util
+import java.util.{HashMap => JavaHashMap, List => JavaList}
+
+import scala.collection.JavaConversions._
+import scala.collection.mutable.ArrayBuffer
+import scala.util.{Failure, Success, Try}
+import scala.util.control.Breaks.{break, breakable}
object YarnUtils extends Logger {
@@ -47,8 +47,7 @@ object YarnUtils extends Logger {
lazy val PROXY_YARN_URL = InternalConfigHolder.get[String](CommonConfig.STREAMPARK_PROXY_YARN_URL)
/**
- * hadoop.http.authentication.type<br>
- * get yarn http authentication mode.<br> ex: sample, kerberos
+ * hadoop.http.authentication.type<br> get yarn http authentication mode.<br> ex: sample, kerberos
*
* @return
*/
@@ -65,7 +64,10 @@ object YarnUtils extends Logger {
val appStates = util.EnumSet.of(RUNNING, ACCEPTED, SUBMITTED)
val appIds =
try {
- HadoopUtils.yarnClient.getApplications(appStates).filter(_.getName == appName).map(_.getApplicationId)
+ HadoopUtils.yarnClient
+ .getApplications(appStates)
+ .filter(_.getName == appName)
+ .map(_.getApplicationId)
} catch {
case e: Exception =>
e.printStackTrace()
@@ -97,7 +99,8 @@ object YarnUtils extends Logger {
/**
* Determine if the task named appName is running in yarn
*
- * @return boolean
+ * @return
+ * boolean
* @param appName
* @return
*/
@@ -118,7 +121,7 @@ object YarnUtils extends Logger {
* <pre>
*
* @return
- * </pre>
+ * </pre>
*/
def getRMWebAppURL(): String = {
@@ -134,60 +137,74 @@ object YarnUtils extends Logger {
rmHttpURL = Option(conf.get("yarn.web-proxy.address", null)) match {
case Some(proxy) => s"$protocol$proxy"
case _ =>
- val name = if (!HAUtil.isHAEnabled(conf)) addressPrefix
- else {
- val yarnConf = new YarnConfiguration(conf)
- val activeRMId = {
- Option(RMHAUtils.findActiveRMHAId(yarnConf)) match {
- case Some(x) =>
- logInfo("findActiveRMHAId successful")
- x
- case None =>
- // if you don't know why, don't modify it
- logWarn(s"findActiveRMHAId is null,config yarn.acl.enable:${yarnConf.get("yarn.acl.enable")},now http try it.")
- // url ==> rmId
- val idUrlMap = new JavaHashMap[String, String]
- val rmIds = HAUtil.getRMHAIds(conf)
- rmIds.foreach(id => {
- val address = conf.get(HAUtil.addSuffix(addressPrefix, id)) match {
- case null =>
- val hostname = conf.get(HAUtil.addSuffix("yarn.resourcemanager.hostname", id))
- s"$hostname:$defaultPort"
- case x => x
- }
- idUrlMap.put(s"$protocol$address", id)
- })
- var rmId: String = null
- val rpcTimeoutForChecks = yarnConf.getInt(
- CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_KEY,
- CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_DEFAULT)
- breakable(idUrlMap.foreach(x => {
- // test yarn url
- val activeUrl = httpTestYarnRMUrl(x._1, rpcTimeoutForChecks)
- if (activeUrl != null) {
- rmId = idUrlMap(activeUrl)
- break
- }
- }))
- rmId
+ val name =
+ if (!HAUtil.isHAEnabled(conf)) addressPrefix
+ else {
+ val yarnConf = new YarnConfiguration(conf)
+ val activeRMId = {
+ Option(RMHAUtils.findActiveRMHAId(yarnConf)) match {
+ case Some(x) =>
+ logInfo("findActiveRMHAId successful")
+ x
+ case None =>
+ // if you don't know why, don't modify it
+ logWarn(
+ s"findActiveRMHAId is null,config yarn.acl.enable:${yarnConf.get("yarn.acl.enable")},now http try it.")
+ // url ==> rmId
+ val idUrlMap = new JavaHashMap[String, String]
+ val rmIds = HAUtil.getRMHAIds(conf)
+ rmIds.foreach(
+ id => {
+ val address = conf.get(HAUtil.addSuffix(addressPrefix, id)) match {
+ case null =>
+ val hostname =
+ conf.get(HAUtil.addSuffix("yarn.resourcemanager.hostname", id))
+ s"$hostname:$defaultPort"
+ case x => x
+ }
+ idUrlMap.put(s"$protocol$address", id)
+ })
+ var rmId: String = null
+ val rpcTimeoutForChecks = yarnConf.getInt(
+ CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_KEY,
+ CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_DEFAULT)
+ breakable(
+ idUrlMap.foreach(
+ x => {
+ // test yarn url
+ val activeUrl = httpTestYarnRMUrl(x._1, rpcTimeoutForChecks)
+ if (activeUrl != null) {
+ rmId = idUrlMap(activeUrl)
+ break
+ }
+ }))
+ rmId
+ }
+ }
+ require(
+ activeRMId != null,
+ "[StreamPark] YarnUtils.getRMWebAppURL: can not found yarn active node")
+ logInfo(s"current activeRMHAId: $activeRMId")
+ val appActiveRMKey = HAUtil.addSuffix(addressPrefix, activeRMId)
+ val hostnameActiveRMKey =
+ HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, activeRMId)
+ if (
+ null == HAUtil.getConfValueForRMInstance(
+ appActiveRMKey,
+ yarnConf) && null != HAUtil.getConfValueForRMInstance(
+ hostnameActiveRMKey,
+ yarnConf)
+ ) {
+ logInfo(s"Find rm web address by : $hostnameActiveRMKey")
+ hostnameActiveRMKey
+ } else {
+ logInfo(s"Find rm web address by : $appActiveRMKey")
+ appActiveRMKey
}
}
- require(activeRMId != null, "[StreamPark] YarnUtils.getRMWebAppURL: can not found yarn active node")
- logInfo(s"current activeRMHAId: $activeRMId")
- val appActiveRMKey = HAUtil.addSuffix(addressPrefix, activeRMId)
- val hostnameActiveRMKey = HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, activeRMId)
- if (null == HAUtil.getConfValueForRMInstance(appActiveRMKey, yarnConf) && null != HAUtil.getConfValueForRMInstance(
- hostnameActiveRMKey,
- yarnConf)) {
- logInfo(s"Find rm web address by : $hostnameActiveRMKey")
- hostnameActiveRMKey
- } else {
- logInfo(s"Find rm web address by : $appActiveRMKey")
- appActiveRMKey
- }
- }
- val inetSocketAddress = conf.getSocketAddr(name, s"0.0.0.0:$defaultPort", defaultPort.toInt)
+ val inetSocketAddress =
+ conf.getSocketAddr(name, s"0.0.0.0:$defaultPort", defaultPort.toInt)
val address = NetUtils.getConnectAddress(inetSocketAddress)
@@ -228,10 +245,12 @@ object YarnUtils extends Logger {
}
}
- def getYarnAppTrackingUrl(applicationId: ApplicationId): String = HadoopUtils.yarnClient.getApplicationReport(applicationId).getTrackingUrl
+ def getYarnAppTrackingUrl(applicationId: ApplicationId): String =
+ HadoopUtils.yarnClient.getApplicationReport(applicationId).getTrackingUrl
/**
- * @param url url
+ * @param url
+ * url
* @return
*/
def restRequest(url: String): String = {
@@ -241,16 +260,18 @@ object YarnUtils extends Logger {
logDebug("request url is " + url);
val config = RequestConfig.custom.setConnectTimeout(5000).build
if (hasYarnHttpKerberosAuth) {
- HadoopUtils.getUgi().doAs(new PrivilegedExceptionAction[String] {
- override def run(): String = {
- Try(HttpClientUtils.httpAuthGetRequest(url, config)) match {
- case Success(v) => v
- case Failure(e) =>
- logError("yarnUtils authRestRequest error, detail: ", e)
- null
+ HadoopUtils
+ .getUgi()
+ .doAs(new PrivilegedExceptionAction[String] {
+ override def run(): String = {
+ Try(HttpClientUtils.httpAuthGetRequest(url, config)) match {
+ case Success(v) => v
+ case Failure(e) =>
+ logError("yarnUtils authRestRequest error, detail: ", e)
+ null
+ }
}
- }
- })
+ })
} else {
Try(HttpClientUtils.httpGetRequest(url, config)) match {
case Success(v) => v
diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/ZooKeeperUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/ZooKeeperUtils.scala
index acab0b688..b8cd19c28 100644
--- a/streampark-common/src/main/scala/org/apache/streampark/common/util/ZooKeeperUtils.scala
+++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/ZooKeeperUtils.scala
@@ -16,16 +16,16 @@
*/
package org.apache.streampark.common.util
-import java.nio.charset.StandardCharsets
-
-import scala.collection.JavaConversions._
-import scala.collection.mutable
-
import org.apache.curator.RetryPolicy
import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory}
import org.apache.curator.retry.RetryNTimes
import org.apache.zookeeper.CreateMode
+import java.nio.charset.StandardCharsets
+
+import scala.collection.JavaConversions._
+import scala.collection.mutable
+
object ZooKeeperUtils {
private[this] val connect = "localhost:2181"
@@ -48,11 +48,11 @@ object ZooKeeperUtils {
// Curator link zookeeper strategy: RetryUntilElapsed maxElapsedTimeMs: Maximum retry time sleepMsBetweenRetries: After each retry interval, after the retry time exceeds maxElapsedTimeMs, it will not retry
// val retryPolicy:RetryPolicy = new RetryUntilElapsed(2000, 3000)
val retryPolicy: RetryPolicy = new RetryNTimes(5, 2000)
- val client = CuratorFrameworkFactory
- .builder
+ val client = CuratorFrameworkFactory.builder
.connectString(url)
.retryPolicy(retryPolicy)
- .connectionTimeoutMs(2000).build
+ .connectionTimeoutMs(2000)
+ .build
client.start()
map += url -> client
client
@@ -79,7 +79,11 @@ object ZooKeeperUtils {
}
}
- def create(path: String, value: String = null, url: String = connect, persistent: Boolean = false): Boolean = {
+ def create(
+ path: String,
+ value: String = null,
+ url: String = connect,
+ persistent: Boolean = false): Boolean = {
try {
val client = getClient(url)
val stat = client.checkExists.forPath(path)
@@ -90,7 +94,8 @@ object ZooKeeperUtils {
case _ => value.getBytes(StandardCharsets.UTF_8)
}
val mode = if (persistent) CreateMode.PERSISTENT else CreateMode.EPHEMERAL
- val opResult = client.create().creatingParentsIfNeeded().withMode(mode).forPath(path, data)
+ val opResult =
+ client.create().creatingParentsIfNeeded().withMode(mode).forPath(path, data)
path == opResult
case _ => false
}
@@ -101,14 +106,20 @@ object ZooKeeperUtils {
}
}
- def update(path: String, value: String, url: String = connect, persistent: Boolean = false): Boolean = {
+ def update(
+ path: String,
+ value: String,
+ url: String = connect,
+ persistent: Boolean = false): Boolean = {
try {
val client = getClient(url)
val stat = client.checkExists.forPath(path)
stat match {
case null =>
val mode = if (persistent) CreateMode.PERSISTENT else CreateMode.EPHEMERAL
- val opResult = client.create.creatingParentsIfNeeded.withMode(mode).forPath(path, value.getBytes(StandardCharsets.UTF_8))
+ val opResult = client.create.creatingParentsIfNeeded
+ .withMode(mode)
+ .forPath(path, value.getBytes(StandardCharsets.UTF_8))
path == opResult
case _ =>
val opResult = client.setData().forPath(path, value.getBytes(StandardCharsets.UTF_8))
diff --git a/streampark-common/src/test/scala/org/apache/streampark/common/fs/LfsOperatorTest.scala b/streampark-common/src/test/scala/org/apache/streampark/common/fs/LfsOperatorTest.scala
index 8960d2be1..6fd1f218b 100644
--- a/streampark-common/src/test/scala/org/apache/streampark/common/fs/LfsOperatorTest.scala
+++ b/streampark-common/src/test/scala/org/apache/streampark/common/fs/LfsOperatorTest.scala
@@ -16,17 +16,17 @@
*/
package org.apache.streampark.common.fs
-import java.io.{File, FileInputStream}
-import java.nio.file.{Files, Paths}
-
-import scala.language.implicitConversions
+import org.apache.streampark.common.fs.LfsOperatorTest.withTempDir
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.io.{FileUtils, IOUtils}
import org.junit.jupiter.api.Assertions._
import org.junit.jupiter.api.Test
-import org.apache.streampark.common.fs.LfsOperatorTest.withTempDir
+import java.io.{File, FileInputStream}
+import java.nio.file.{Files, Paths}
+
+import scala.language.implicitConversions
object LfsOperatorTest {
@@ -44,262 +44,277 @@ object LfsOperatorTest {
class LfsOperatorTest {
@Test
- def testMkdirs(): Unit = withTempDir { outputDir =>
- assertDoesNotThrow(LfsOperator.mkdirs(null))
- assertDoesNotThrow(LfsOperator.mkdirs(""))
- assertTrue(LfsOperator.exists(outputDir))
-
- // duplicate mkdirs
- assertDoesNotThrow {
- LfsOperator.mkdirs(s"$outputDir/test")
- LfsOperator.mkdirs(s"$outputDir/test")
- }
+ def testMkdirs(): Unit = withTempDir {
+ outputDir =>
+ assertDoesNotThrow(LfsOperator.mkdirs(null))
+ assertDoesNotThrow(LfsOperator.mkdirs(""))
+ assertTrue(LfsOperator.exists(outputDir))
+
+ // duplicate mkdirs
+ assertDoesNotThrow {
+ LfsOperator.mkdirs(s"$outputDir/test")
+ LfsOperator.mkdirs(s"$outputDir/test")
+ }
}
@Test
- def testExists(): Unit = withTempDir { outputDir =>
- assertDoesNotThrow {
- val dir = s"$outputDir/tmp"
- val f = new File(dir)
- f.mkdirs
- assertTrue(LfsOperator.exists(f.getAbsolutePath))
- assertTrue(LfsOperator.exists(dir))
- }
+ def testExists(): Unit = withTempDir {
+ outputDir =>
+ assertDoesNotThrow {
+ val dir = s"$outputDir/tmp"
+ val f = new File(dir)
+ f.mkdirs
+ assertTrue(LfsOperator.exists(f.getAbsolutePath))
+ assertTrue(LfsOperator.exists(dir))
+ }
- // path that does not exists
- assertFalse(LfsOperator.exists(null))
- assertFalse(LfsOperator.exists(""))
- assertFalse(LfsOperator.exists(s"$outputDir/233"))
+ // path that does not exists
+ assertFalse(LfsOperator.exists(null))
+ assertFalse(LfsOperator.exists(""))
+ assertFalse(LfsOperator.exists(s"$outputDir/233"))
}
@Test
- def testMkCleanDirs(): Unit = withTempDir { outputDir =>
- assertDoesNotThrow {
- Array.fill(5)(genRandomFile(outputDir))
- assertEquals(new File(outputDir).list.length, 5)
- LfsOperator.mkCleanDirs(outputDir)
- val dir = new File(outputDir)
- assertTrue(dir.exists)
- assertTrue(dir.isDirectory)
- assertEquals(dir.list.length, 0)
- }
+ def testMkCleanDirs(): Unit = withTempDir {
+ outputDir =>
+ assertDoesNotThrow {
+ Array.fill(5)(genRandomFile(outputDir))
+ assertEquals(new File(outputDir).list.length, 5)
+ LfsOperator.mkCleanDirs(outputDir)
+ val dir = new File(outputDir)
+ assertTrue(dir.exists)
+ assertTrue(dir.isDirectory)
+ assertEquals(dir.list.length, 0)
+ }
- // path that does not exists
- assertDoesNotThrow(LfsOperator.mkdirs(null))
- assertDoesNotThrow(LfsOperator.mkdirs(""))
+ // path that does not exists
+ assertDoesNotThrow(LfsOperator.mkdirs(null))
+ assertDoesNotThrow(LfsOperator.mkdirs(""))
- // clean dirs that does not exists
- assertTrue {
- LfsOperator.mkCleanDirs(s"$outputDir/114514")
- new File(s"$outputDir/114514").exists
- }
+ // clean dirs that does not exists
+ assertTrue {
+ LfsOperator.mkCleanDirs(s"$outputDir/114514")
+ new File(s"$outputDir/114514").exists
+ }
}
@Test
- def listDir(): Unit = withTempDir { outputDir =>
- // list directory
- assertTrue {
- val expectFs = genRandomDir(outputDir)._2
- val actualFs = LfsOperator.listDir(outputDir)
- expectFs.map(_.getName).sorted.sameElements(actualFs.map(_.getName).sorted)
- }
+ def listDir(): Unit = withTempDir {
+ outputDir =>
+ // list directory
+ assertTrue {
+ val expectFs = genRandomDir(outputDir)._2
+ val actualFs = LfsOperator.listDir(outputDir)
+ expectFs.map(_.getName).sorted.sameElements(actualFs.map(_.getName).sorted)
+ }
- // list file
- assertTrue {
- val file = genRandomFile(outputDir)
- val listFile = LfsOperator.listDir(file.getAbsolutePath)
- listFile.length == 1 && listFile.head.getName == file.getName
- }
+ // list file
+ assertTrue {
+ val file = genRandomFile(outputDir)
+ val listFile = LfsOperator.listDir(file.getAbsolutePath)
+ listFile.length == 1 && listFile.head.getName == file.getName
+ }
- // path that does not exists
- assertTrue(LfsOperator.listDir("").isEmpty)
- assertTrue(LfsOperator.listDir(null).isEmpty)
- assertTrue(LfsOperator.listDir(s"$outputDir/114514").isEmpty)
+ // path that does not exists
+ assertTrue(LfsOperator.listDir("").isEmpty)
+ assertTrue(LfsOperator.listDir(null).isEmpty)
+ assertTrue(LfsOperator.listDir(s"$outputDir/114514").isEmpty)
}
@Test
- def testDelete(): Unit = withTempDir { outputDir =>
- // delete directory
- assertFalse {
- val dir = s"$outputDir/tmp"
- genRandomDir(dir)
- LfsOperator.delete(dir)
- new File(dir).exists
- }
- // delete file
- assertFalse {
- val file = genRandomFile(outputDir)
- LfsOperator.delete(file.getAbsolutePath)
- file.exists
- }
- // path that does not exists
- assertDoesNotThrow(LfsOperator.delete(null))
- assertDoesNotThrow(LfsOperator.delete(""))
- assertDoesNotThrow(LfsOperator.delete(s"$outputDir/114514"))
+ def testDelete(): Unit = withTempDir {
+ outputDir =>
+ // delete directory
+ assertFalse {
+ val dir = s"$outputDir/tmp"
+ genRandomDir(dir)
+ LfsOperator.delete(dir)
+ new File(dir).exists
+ }
+ // delete file
+ assertFalse {
+ val file = genRandomFile(outputDir)
+ LfsOperator.delete(file.getAbsolutePath)
+ file.exists
+ }
+ // path that does not exists
+ assertDoesNotThrow(LfsOperator.delete(null))
+ assertDoesNotThrow(LfsOperator.delete(""))
+ assertDoesNotThrow(LfsOperator.delete(s"$outputDir/114514"))
}
// noinspection TypeAnnotation
val md5Hex = (f: File) => DigestUtils.md5Hex(IOUtils.toByteArray(new FileInputStream(f)))
// noinspection TypeAnnotation
val sameFilesHex = (f1: Seq[File], f2: Seq[File]) =>
- f1.map(_.getName).sorted == f2.map(_.getName).sorted && f1.map(md5Hex).sorted == f2.map(md5Hex).sorted
+ f1.map(_.getName).sorted == f2.map(_.getName).sorted && f1.map(md5Hex).sorted == f2
+ .map(md5Hex)
+ .sorted
@Test
- def testCopy(): Unit = withTempDir { outputDir =>
- // copy file to file path or directory path
- assertDoesNotThrow {
- val file = genRandomFile(outputDir)
-
- def assertCopy(to: String, expectedOut: String): Unit = {
- LfsOperator.copy(file.getAbsolutePath, to)
- val output = new File(expectedOut)
- assertTrue(output.exists)
- assertTrue(file.length() == output.length())
+ def testCopy(): Unit = withTempDir {
+ outputDir =>
+ // copy file to file path or directory path
+ assertDoesNotThrow {
+ val file = genRandomFile(outputDir)
+
+ def assertCopy(to: String, expectedOut: String): Unit = {
+ LfsOperator.copy(file.getAbsolutePath, to)
+ val output = new File(expectedOut)
+ assertTrue(output.exists)
+ assertTrue(file.length() == output.length())
+ }
+
+ Files.createDirectory(Paths.get(outputDir, "out-1"))
+ assertCopy(s"$outputDir/out-1", s"$outputDir/out-1/${file.getName}")
+ Files.createDirectory(Paths.get(outputDir, "out-2"))
+ assertCopy(s"$outputDir/out-2/${file.getName}", s"$outputDir/out-2/${file.getName}")
+ Files.createDirectory(Paths.get(outputDir, "out-3"))
+ assertCopy(s"$outputDir/out-3/114514.dat", s"$outputDir/out-3/114514.dat")
}
- Files.createDirectory(Paths.get(outputDir, "out-1"))
- assertCopy(s"$outputDir/out-1", s"$outputDir/out-1/${file.getName}")
- Files.createDirectory(Paths.get(outputDir, "out-2"))
- assertCopy(s"$outputDir/out-2/${file.getName}", s"$outputDir/out-2/${file.getName}")
- Files.createDirectory(Paths.get(outputDir, "out-3"))
- assertCopy(s"$outputDir/out-3/114514.dat", s"$outputDir/out-3/114514.dat")
- }
-
- // copy file that not exists
- assertDoesNotThrow {
- LfsOperator.copy(s"$outputDir/nobody.dat", s"$outputDir/out-5/nobody.dat")
- assertFalse(new File(s"$outputDir/out-5/nobody.dat").exists)
- }
+ // copy file that not exists
+ assertDoesNotThrow {
+ LfsOperator.copy(s"$outputDir/nobody.dat", s"$outputDir/out-5/nobody.dat")
+ assertFalse(new File(s"$outputDir/out-5/nobody.dat").exists)
+ }
- // copy directory
- val dir = genRandomDir(outputDir.concat("/in-1"))._1
- assertThrows(classOf[IllegalArgumentException], LfsOperator.copy(dir.getAbsolutePath, s"$outputDir/out-6"))
-
- // delete or not delete the original file
- assertDoesNotThrow {
- // not delete origin file
- val file = genRandomFile(outputDir)
- LfsOperator.copy(file.getAbsolutePath, s"$outputDir/out-7", delSrc = false)
- assertTrue(file.exists)
- // delete origin file
- LfsOperator.copy(file.getAbsolutePath, s"$outputDir/out-8", delSrc = true)
- assertFalse(file.exists)
- }
+ // copy directory
+ val dir = genRandomDir(outputDir.concat("/in-1"))._1
+ assertThrows(
+ classOf[IllegalArgumentException],
+ LfsOperator.copy(dir.getAbsolutePath, s"$outputDir/out-6"))
+
+ // delete or not delete the original file
+ assertDoesNotThrow {
+ // not delete origin file
+ val file = genRandomFile(outputDir)
+ LfsOperator.copy(file.getAbsolutePath, s"$outputDir/out-7", delSrc = false)
+ assertTrue(file.exists)
+ // delete origin file
+ LfsOperator.copy(file.getAbsolutePath, s"$outputDir/out-8", delSrc = true)
+ assertFalse(file.exists)
+ }
- // overwritten or non-overwritten copy
- val file = genRandomFile(outputDir, "114514-233.dat")
- // overwritten
- assertDoesNotThrow {
- val out = genRandomFile(s"$outputDir/out-9", "114514-233.dat")
- val md5Before = md5Hex(out)
- LfsOperator.copy(file.getAbsolutePath, out.getAbsolutePath, overwrite = true)
- val md5After = md5Hex(new File(out.getAbsolutePath))
- assertNotEquals(md5Before, md5After)
- assertEquals(md5After, md5Hex(file))
- }
- // non-overwritten
- assertDoesNotThrow {
- val out = genRandomFile(s"$outputDir/out-10", "114514-233.dat")
- val md5Before = md5Hex(out)
- LfsOperator.copy(file.getAbsolutePath, out.getAbsolutePath, overwrite = false)
- val md5After = md5Hex(new File(out.getAbsolutePath))
- assertEquals(md5Before, md5After)
- assertNotEquals(md5After, md5Hex(file))
- }
+ // overwritten or non-overwritten copy
+ val file = genRandomFile(outputDir, "114514-233.dat")
+ // overwritten
+ assertDoesNotThrow {
+ val out = genRandomFile(s"$outputDir/out-9", "114514-233.dat")
+ val md5Before = md5Hex(out)
+ LfsOperator.copy(file.getAbsolutePath, out.getAbsolutePath, overwrite = true)
+ val md5After = md5Hex(new File(out.getAbsolutePath))
+ assertNotEquals(md5Before, md5After)
+ assertEquals(md5After, md5Hex(file))
+ }
+ // non-overwritten
+ assertDoesNotThrow {
+ val out = genRandomFile(s"$outputDir/out-10", "114514-233.dat")
+ val md5Before = md5Hex(out)
+ LfsOperator.copy(file.getAbsolutePath, out.getAbsolutePath, overwrite = false)
+ val md5After = md5Hex(new File(out.getAbsolutePath))
+ assertEquals(md5Before, md5After)
+ assertNotEquals(md5After, md5Hex(file))
+ }
}
@Test
- def testCopyDir(): Unit = withTempDir { outputDir =>
- // copy dir
- assertDoesNotThrow {
- val sourceDir = genRandomDir(s"$outputDir/in-1")._1
- val target = s"$outputDir/out-1"
- LfsOperator.copyDir(sourceDir.getAbsolutePath, target)
- val targetDir = new File(target)
- assertTrue(targetDir.exists)
- assertTrue(targetDir.isDirectory)
- assertTrue(sameFilesHex(sourceDir.listFiles, targetDir.listFiles))
- }
+ def testCopyDir(): Unit = withTempDir {
+ outputDir =>
+ // copy dir
+ assertDoesNotThrow {
+ val sourceDir = genRandomDir(s"$outputDir/in-1")._1
+ val target = s"$outputDir/out-1"
+ LfsOperator.copyDir(sourceDir.getAbsolutePath, target)
+ val targetDir = new File(target)
+ assertTrue(targetDir.exists)
+ assertTrue(targetDir.isDirectory)
+ assertTrue(sameFilesHex(sourceDir.listFiles, targetDir.listFiles))
+ }
- // copy directory that not exists
- assertDoesNotThrow {
- LfsOperator.copyDir(s"$outputDir/in-2", s"$outputDir/out-2")
- assertFalse(new File(s"$outputDir/out-2").exists)
- LfsOperator.copyDir("", s"$outputDir/out-2")
- LfsOperator.copyDir(null, s"$outputDir/out-2")
- }
+ // copy directory that not exists
+ assertDoesNotThrow {
+ LfsOperator.copyDir(s"$outputDir/in-2", s"$outputDir/out-2")
+ assertFalse(new File(s"$outputDir/out-2").exists)
+ LfsOperator.copyDir("", s"$outputDir/out-2")
+ LfsOperator.copyDir(null, s"$outputDir/out-2")
+ }
- // copy file
- assertThrows(
- classOf[IllegalArgumentException],
- LfsOperator.copyDir(genRandomFile(outputDir).getAbsolutePath, s"$outputDir/out-3"))
-
- // delete or not delete the original dir
- assertDoesNotThrow {
- // not delete origin dir
- val sourceDir = genRandomDir(s"$outputDir/in-4")._1
- LfsOperator.copyDir(sourceDir.getAbsolutePath, s"$outputDir/out-4", delSrc = false)
- assertTrue(sourceDir.exists)
- // delete origin dir
- LfsOperator.copyDir(sourceDir.getAbsolutePath, s"$outputDir/out-5", delSrc = true)
- assertFalse(sourceDir.exists)
- }
+ // copy file
+ assertThrows(
+ classOf[IllegalArgumentException],
+ LfsOperator.copyDir(genRandomFile(outputDir).getAbsolutePath, s"$outputDir/out-3"))
+
+ // delete or not delete the original dir
+ assertDoesNotThrow {
+ // not delete origin dir
+ val sourceDir = genRandomDir(s"$outputDir/in-4")._1
+ LfsOperator.copyDir(sourceDir.getAbsolutePath, s"$outputDir/out-4", delSrc = false)
+ assertTrue(sourceDir.exists)
+ // delete origin dir
+ LfsOperator.copyDir(sourceDir.getAbsolutePath, s"$outputDir/out-5", delSrc = true)
+ assertFalse(sourceDir.exists)
+ }
}
@Test
- def testMove(): Unit = withTempDir { outputDir =>
- // move file to directory
- assertDoesNotThrow {
- val sourceFile = genRandomFile(outputDir)
- val sourceMd5 = DigestUtils.md5Hex(IOUtils.toByteArray(new FileInputStream(sourceFile)))
- val targetPath = s"$outputDir/target-1"
- LfsOperator.move(sourceFile.getAbsolutePath, targetPath)
-
- val targetFile = new File(targetPath, sourceFile.getName)
- assertTrue(targetFile.exists)
- assertEquals(sourceMd5, DigestUtils.md5Hex(IOUtils.toByteArray(new FileInputStream(targetFile))))
- }
+ def testMove(): Unit = withTempDir {
+ outputDir =>
+ // move file to directory
+ assertDoesNotThrow {
+ val sourceFile = genRandomFile(outputDir)
+ val sourceMd5 = DigestUtils.md5Hex(IOUtils.toByteArray(new FileInputStream(sourceFile)))
+ val targetPath = s"$outputDir/target-1"
+ LfsOperator.move(sourceFile.getAbsolutePath, targetPath)
+
+ val targetFile = new File(targetPath, sourceFile.getName)
+ assertTrue(targetFile.exists)
+ assertEquals(
+ sourceMd5,
+ DigestUtils.md5Hex(IOUtils.toByteArray(new FileInputStream(targetFile))))
+ }
- // move directory to directory
- assertDoesNotThrow {
- val (sourceDir, sourceFiles) = genRandomDir(s"$outputDir/tmp")
- val targetPath = s"$outputDir/target-2"
- LfsOperator.move(sourceDir.getAbsolutePath, targetPath)
- val targetDir = new File(targetPath.concat("/tmp"))
- assertTrue(targetDir.exists)
- assertTrue(sourceFiles.map(_.getName).sorted.sameElements(targetDir.listFiles.map(_.getName).sorted))
- }
+ // move directory to directory
+ assertDoesNotThrow {
+ val (sourceDir, sourceFiles) = genRandomDir(s"$outputDir/tmp")
+ val targetPath = s"$outputDir/target-2"
+ LfsOperator.move(sourceDir.getAbsolutePath, targetPath)
+ val targetDir = new File(targetPath.concat("/tmp"))
+ assertTrue(targetDir.exists)
+ assertTrue(
+ sourceFiles.map(_.getName).sorted.sameElements(targetDir.listFiles.map(_.getName).sorted))
+ }
- // file that not exists
- assertFalse {
- LfsOperator.move(s"$outputDir/aha.dat", s"$outputDir/target-3")
- new File(s"$outputDir/target-3/aha.dat").exists
- }
- assertDoesNotThrow {
- val file = genRandomFile(outputDir)
- assertDoesNotThrow(LfsOperator.move(file.getAbsolutePath, null))
- assertDoesNotThrow(LfsOperator.move(file.getAbsolutePath, ""))
- }
+ // file that not exists
+ assertFalse {
+ LfsOperator.move(s"$outputDir/aha.dat", s"$outputDir/target-3")
+ new File(s"$outputDir/target-3/aha.dat").exists
+ }
+ assertDoesNotThrow {
+ val file = genRandomFile(outputDir)
+ assertDoesNotThrow(LfsOperator.move(file.getAbsolutePath, null))
+ assertDoesNotThrow(LfsOperator.move(file.getAbsolutePath, ""))
+ }
- // duplicate move file
- assertDoesNotThrow {
- val file = genRandomFile(outputDir)
- val target = s"$outputDir/target-4"
- LfsOperator.move(file.getAbsolutePath, target)
- LfsOperator.move(file.getAbsolutePath, target)
- }
+ // duplicate move file
+ assertDoesNotThrow {
+ val file = genRandomFile(outputDir)
+ val target = s"$outputDir/target-4"
+ LfsOperator.move(file.getAbsolutePath, target)
+ LfsOperator.move(file.getAbsolutePath, target)
+ }
- // duplicate move directory
- assertDoesNotThrow {
- val dir = genRandomDir(s"$outputDir/tmp-5", 3)._1
- val target = s"$outputDir/target-5"
- LfsOperator.move(dir.getAbsolutePath, target)
- LfsOperator.move(dir.getAbsolutePath, target)
- assertTrue(new File(s"$target/tmp-5").exists)
- assertEquals(new File(s"$target/tmp-5").listFiles.length, 3)
- }
+ // duplicate move directory
+ assertDoesNotThrow {
+ val dir = genRandomDir(s"$outputDir/tmp-5", 3)._1
+ val target = s"$outputDir/target-5"
+ LfsOperator.move(dir.getAbsolutePath, target)
+ LfsOperator.move(dir.getAbsolutePath, target)
+ assertTrue(new File(s"$target/tmp-5").exists)
+ assertEquals(new File(s"$target/tmp-5").listFiles.length, 3)
+ }
}
@Test
diff --git a/streampark-common/src/test/scala/org/apache/streampark/common/fs/package.scala b/streampark-common/src/test/scala/org/apache/streampark/common/fs/package.scala
index 00c079765..bffef45b0 100644
--- a/streampark-common/src/test/scala/org/apache/streampark/common/fs/package.scala
+++ b/streampark-common/src/test/scala/org/apache/streampark/common/fs/package.scala
@@ -16,14 +16,14 @@
*/
package org.apache.streampark.common
+import org.apache.commons.io.FileUtils
+import org.junit.jupiter.api.function.Executable
+
import java.io.File
import scala.language.implicitConversions
import scala.util.Random
-import org.apache.commons.io.FileUtils
-import org.junit.jupiter.api.function.Executable
-
package object fs {
implicit def funcToExecutable[R](func: => R): Executable = new Executable() {
@@ -33,10 +33,15 @@ package object fs {
/**
* generate a random binary file
*
- * @param dir parent directory
- * @param size file size
+ * @param dir
+ * parent directory
+ * @param size
+ * file size
*/
- def genRandomFile(dir: String, name: String = s"${java.util.UUID.randomUUID().toString}.dat", size: Int = 256): File = {
+ def genRandomFile(
+ dir: String,
+ name: String = s"${java.util.UUID.randomUUID().toString}.dat",
+ size: Int = 256): File = {
val random = new Random()
val c = new Array[Byte](size)
random.nextBytes(c)
@@ -48,7 +53,8 @@ package object fs {
/**
* generate a random directory that contains some random files
*
- * @return Directory File object -> Children random files array.
+ * @return
+ * Directory File object -> Children random files array.
*/
def genRandomDir(dirPath: String, childFileCount: Int = 5): (File, Array[File]) = {
val dir = new File(dirPath)
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/FlinkClient.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/FlinkClient.scala
index 328995036..c2ea9ba48 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/FlinkClient.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/FlinkClient.scala
@@ -19,25 +19,31 @@ package org.apache.streampark.flink.client
import org.apache.streampark.common.conf.FlinkVersion
import org.apache.streampark.common.util.Logger
-import org.apache.streampark.flink.proxy.FlinkShimsProxy
import org.apache.streampark.flink.client.bean._
+import org.apache.streampark.flink.proxy.FlinkShimsProxy
import scala.language.{implicitConversions, reflectiveCalls}
import scala.reflect.ClassTag
object FlinkClient extends Logger {
- private[this] val FLINK_CLIENT_HANDLER_CLASS_NAME = "org.apache.streampark.flink.client.FlinkClientHandler"
+ private[this] val FLINK_CLIENT_HANDLER_CLASS_NAME =
+ "org.apache.streampark.flink.client.FlinkClientHandler"
- private[this] val SUBMIT_REQUEST = "org.apache.streampark.flink.client.bean.SubmitRequest" -> "submit"
+ private[this] val SUBMIT_REQUEST =
+ "org.apache.streampark.flink.client.bean.SubmitRequest" -> "submit"
- private[this] val DEPLOY_REQUEST = "org.apache.streampark.flink.client.bean.DeployRequest" -> "deploy"
+ private[this] val DEPLOY_REQUEST =
+ "org.apache.streampark.flink.client.bean.DeployRequest" -> "deploy"
- private[this] val CANCEL_REQUEST = "org.apache.streampark.flink.client.bean.CancelRequest" -> "cancel"
+ private[this] val CANCEL_REQUEST =
+ "org.apache.streampark.flink.client.bean.CancelRequest" -> "cancel"
- private[this] val SHUTDOWN_REQUEST = "org.apache.streampark.flink.client.bean.ShutDownRequest" -> "shutdown"
+ private[this] val SHUTDOWN_REQUEST =
+ "org.apache.streampark.flink.client.bean.ShutDownRequest" -> "shutdown"
- private[this] val SAVEPOINT_REQUEST = "org.apache.streampark.flink.client.bean.TriggerSavepointRequest" -> "triggerSavepoint"
+ private[this] val SAVEPOINT_REQUEST =
+ "org.apache.streampark.flink.client.bean.TriggerSavepointRequest" -> "triggerSavepoint"
def submit(submitRequest: SubmitRequest): SubmitResponse = {
proxy[SubmitResponse](submitRequest, submitRequest.flinkVersion, SUBMIT_REQUEST)
@@ -59,18 +65,25 @@ object FlinkClient extends Logger {
proxy[SavepointResponse](savepointRequest, savepointRequest.flinkVersion, SAVEPOINT_REQUEST)
}
- private[this] def proxy[T: ClassTag](request: Object, flinkVersion: FlinkVersion, requestBody: (String, String)): T = {
+ private[this] def proxy[T: ClassTag](
+ request: Object,
+ flinkVersion: FlinkVersion,
+ requestBody: (String, String)): T = {
flinkVersion.checkVersion()
- FlinkShimsProxy.proxy(flinkVersion, (classLoader: ClassLoader) => {
- val submitClass = classLoader.loadClass(FLINK_CLIENT_HANDLER_CLASS_NAME)
- val requestClass = classLoader.loadClass(requestBody._1)
- val method = submitClass.getDeclaredMethod(requestBody._2, requestClass)
- method.setAccessible(true)
- val obj = method.invoke(null, FlinkShimsProxy.getObject(classLoader, request))
- if (obj == null) null.asInstanceOf[T] else {
- FlinkShimsProxy.getObject[T](this.getClass.getClassLoader, obj)
+ FlinkShimsProxy.proxy(
+ flinkVersion,
+ (classLoader: ClassLoader) => {
+ val submitClass = classLoader.loadClass(FLINK_CLIENT_HANDLER_CLASS_NAME)
+ val requestClass = classLoader.loadClass(requestBody._1)
+ val method = submitClass.getDeclaredMethod(requestBody._2, requestClass)
+ method.setAccessible(true)
+ val obj = method.invoke(null, FlinkShimsProxy.getObject(classLoader, request))
+ if (obj == null) null.asInstanceOf[T]
+ else {
+ FlinkShimsProxy.getObject[T](this.getClass.getClassLoader, obj)
+ }
}
- })
+ )
}
}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/CancelRequest.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/CancelRequest.scala
index 94acfd3c4..99615d5d8 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/CancelRequest.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/CancelRequest.scala
@@ -17,19 +17,22 @@
package org.apache.streampark.flink.client.bean
-import java.util.{Map => JavaMap}
-import javax.annotation.Nullable
-
import org.apache.streampark.common.conf.{FlinkVersion, K8sFlinkConfig}
import org.apache.streampark.common.enums.ExecutionMode
-case class CancelRequest(flinkVersion: FlinkVersion,
- executionMode: ExecutionMode,
- @Nullable properties: JavaMap[String, Any],
- clusterId: String,
- jobId: String,
- override val withSavepoint: Boolean,
- withDrain: Boolean,
- savepointPath: String,
- savePointTimeout: Long,
- override val kubernetesNamespace: String = K8sFlinkConfig.DEFAULT_KUBERNETES_NAMESPACE) extends SavepointRequestTrait
+import javax.annotation.Nullable
+
+import java.util.{Map => JavaMap}
+
+case class CancelRequest(
+ flinkVersion: FlinkVersion,
+ executionMode: ExecutionMode,
+ @Nullable properties: JavaMap[String, Any],
+ clusterId: String,
+ jobId: String,
+ override val withSavepoint: Boolean,
+ withDrain: Boolean,
+ savepointPath: String,
+ savePointTimeout: Long,
+ override val kubernetesNamespace: String = K8sFlinkConfig.DEFAULT_KUBERNETES_NAMESPACE)
+ extends SavepointRequestTrait
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/DeployRequest.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/DeployRequest.scala
index fa05c0d7c..db7bbcd3d 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/DeployRequest.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/DeployRequest.scala
@@ -17,27 +17,30 @@
package org.apache.streampark.flink.client.bean
-import java.io.File
-import java.util.{Map => JavaMap}
-import javax.annotation.Nullable
+import org.apache.streampark.common.conf.{FlinkVersion, Workspace}
+import org.apache.streampark.common.enums.{ExecutionMode, FlinkK8sRestExposedType}
+import org.apache.streampark.flink.util.FlinkUtils
import org.apache.commons.io.FileUtils
import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions
-import org.apache.streampark.common.conf.{FlinkVersion, Workspace}
-import org.apache.streampark.common.enums.{ExecutionMode, FlinkK8sRestExposedType}
-import org.apache.streampark.flink.util.FlinkUtils
+import javax.annotation.Nullable
+
+import java.io.File
+import java.util.{Map => JavaMap}
-case class DeployRequest(flinkVersion: FlinkVersion,
- executionMode: ExecutionMode,
- properties: JavaMap[String, Any],
- clusterId: String,
- @Nullable k8sDeployParam: KubernetesDeployParam) {
+case class DeployRequest(
+ flinkVersion: FlinkVersion,
+ executionMode: ExecutionMode,
+ properties: JavaMap[String, Any],
+ clusterId: String,
+ @Nullable k8sDeployParam: KubernetesDeployParam) {
private[client] lazy val hdfsWorkspace = {
/**
- * You must keep the flink version and configuration in the native flink and hdfs exactly the same.
+ * You must keep the flink version and configuration in the native flink and hdfs exactly the
+ * same.
*/
val workspace = Workspace.remote
val flinkHome = flinkVersion.flinkHome
@@ -55,13 +58,15 @@ case class DeployRequest(flinkVersion: FlinkVersion,
flinkPlugins = s"$flinkHdfsHome/plugins",
flinkDistJar = FlinkUtils.getFlinkDistJar(flinkHome),
appJars = workspace.APP_JARS,
- appPlugins = workspace.APP_PLUGINS)
+ appPlugins = workspace.APP_PLUGINS
+ )
}
}
-case class KubernetesDeployParam(clusterId: String,
- kubernetesNamespace: String = KubernetesConfigOptions.NAMESPACE.defaultValue(),
- kubeConf: String = "~/.kube/config",
- serviceAccount: String = KubernetesConfigOptions.KUBERNETES_SERVICE_ACCOUNT.defaultValue(),
- flinkImage: String = KubernetesConfigOptions.CONTAINER_IMAGE.defaultValue(),
- @Nullable flinkRestExposedType: FlinkK8sRestExposedType = FlinkK8sRestExposedType.CLUSTER_IP)
+case class KubernetesDeployParam(
+ clusterId: String,
+ kubernetesNamespace: String = KubernetesConfigOptions.NAMESPACE.defaultValue(),
+ kubeConf: String = "~/.kube/config",
+ serviceAccount: String = KubernetesConfigOptions.KUBERNETES_SERVICE_ACCOUNT.defaultValue(),
+ flinkImage: String = KubernetesConfigOptions.CONTAINER_IMAGE.defaultValue(),
+ @Nullable flinkRestExposedType: FlinkK8sRestExposedType = FlinkK8sRestExposedType.CLUSTER_IP)
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SavepointRequestTrait.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SavepointRequestTrait.scala
index 37ccc069f..11e3866cf 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SavepointRequestTrait.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SavepointRequestTrait.scala
@@ -17,13 +17,13 @@
package org.apache.streampark.flink.client.bean
-import java.util.{Map => JavaMap}
-
import org.apache.streampark.common.conf.{FlinkVersion, K8sFlinkConfig}
import org.apache.streampark.common.enums.ExecutionMode
import javax.annotation.Nullable
+import java.util.{Map => JavaMap}
+
trait SavepointRequestTrait {
val flinkVersion: FlinkVersion
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/ShutDownRequest.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/ShutDownRequest.scala
index c8da072e7..a6b75bbdf 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/ShutDownRequest.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/ShutDownRequest.scala
@@ -17,14 +17,16 @@
package org.apache.streampark.flink.client.bean
-import java.util.{Map => JavaMap}
-import javax.annotation.Nullable
-
import org.apache.streampark.common.conf.FlinkVersion
import org.apache.streampark.common.enums.ExecutionMode
-case class ShutDownRequest(flinkVersion: FlinkVersion,
- executionMode: ExecutionMode,
- @Nullable properties: JavaMap[String, Any],
- clusterId: String,
- @Nullable kubernetesDeployParam: KubernetesDeployParam)
+import javax.annotation.Nullable
+
+import java.util.{Map => JavaMap}
+
+case class ShutDownRequest(
+ flinkVersion: FlinkVersion,
+ executionMode: ExecutionMode,
+ @Nullable properties: JavaMap[String, Any],
+ clusterId: String,
+ @Nullable kubernetesDeployParam: KubernetesDeployParam)
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SubmitRequest.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SubmitRequest.scala
index 0740302d3..eaae692e9 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SubmitRequest.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SubmitRequest.scala
@@ -17,17 +17,6 @@
package org.apache.streampark.flink.client.bean
-import java.io.File
-import java.util.{Map => JavaMap}
-import javax.annotation.Nullable
-
-import scala.collection.JavaConversions._
-import scala.util.Try
-
-import com.fasterxml.jackson.databind.ObjectMapper
-import org.apache.commons.io.FileUtils
-import org.apache.flink.runtime.jobgraph.{SavepointConfigOptions, SavepointRestoreSettings}
-
import org.apache.streampark.common.conf.{ConfigConst, FlinkVersion, Workspace}
import org.apache.streampark.common.conf.ConfigConst._
import org.apache.streampark.common.enums._
@@ -35,28 +24,47 @@ import org.apache.streampark.common.util.{DeflaterUtils, HdfsUtils, PropertiesUt
import org.apache.streampark.flink.packer.pipeline.{BuildResult, ShadedBuildResponse}
import org.apache.streampark.flink.util.FlinkUtils
+import com.fasterxml.jackson.databind.ObjectMapper
+import org.apache.commons.io.FileUtils
+import org.apache.flink.runtime.jobgraph.{SavepointConfigOptions, SavepointRestoreSettings}
+
+import javax.annotation.Nullable
+
+import java.io.File
+import java.util.{Map => JavaMap}
+
+import scala.collection.JavaConversions._
+import scala.util.Try
+
/**
- * @param clusterId flink cluster id in k8s cluster.
- * @param kubernetesNamespace k8s namespace.
- * @param flinkRestExposedType flink rest-service exposed type on k8s cluster.
+ * @param clusterId
+ * flink cluster id in k8s cluster.
+ * @param kubernetesNamespace
+ * k8s namespace.
+ * @param flinkRestExposedType
+ * flink rest-service exposed type on k8s cluster.
*/
-case class KubernetesSubmitParam(clusterId: String, kubernetesNamespace: String, @Nullable flinkRestExposedType: FlinkK8sRestExposedType)
-
-case class SubmitRequest(flinkVersion: FlinkVersion,
- executionMode: ExecutionMode,
- properties: JavaMap[String, Any],
- flinkYaml: String,
- developmentMode: DevelopmentMode,
- id: Long,
- jobId: String,
- appName: String,
- appConf: String,
- applicationType: ApplicationType,
- savePoint: String,
- args: String,
- @Nullable buildResult: BuildResult,
- @Nullable k8sSubmitParam: KubernetesSubmitParam,
- @Nullable extraParameter: JavaMap[String, Any]) {
+case class KubernetesSubmitParam(
+ clusterId: String,
+ kubernetesNamespace: String,
+ @Nullable flinkRestExposedType: FlinkK8sRestExposedType)
+
+case class SubmitRequest(
+ flinkVersion: FlinkVersion,
+ executionMode: ExecutionMode,
+ properties: JavaMap[String, Any],
+ flinkYaml: String,
+ developmentMode: DevelopmentMode,
+ id: Long,
+ jobId: String,
+ appName: String,
+ appConf: String,
+ applicationType: ApplicationType,
+ savePoint: String,
+ args: String,
+ @Nullable buildResult: BuildResult,
+ @Nullable k8sSubmitParam: KubernetesSubmitParam,
+ @Nullable extraParameter: JavaMap[String, Any]) {
lazy val appProperties: Map[String, String] = getParameterMap(KEY_FLINK_PROPERTY_PREFIX)
@@ -67,11 +75,14 @@ case class SubmitRequest(flinkVersion: FlinkVersion,
case _ => appProperties(KEY_FLINK_APPLICATION_MAIN_CLASS)
}
- lazy val effectiveAppName: String = if (this.appName == null) appProperties(KEY_FLINK_APP_NAME) else this.appName
+ lazy val effectiveAppName: String =
+ if (this.appName == null) appProperties(KEY_FLINK_APP_NAME) else this.appName
lazy val flinkSQL: String = extraParameter.get(KEY_FLINK_SQL()).toString
- lazy val allowNonRestoredState = Try(properties.get(SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE.key).toString.toBoolean).getOrElse(false)
+ lazy val allowNonRestoredState = Try(
+ properties.get(SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE.key).toString.toBoolean)
+ .getOrElse(false)
lazy val savepointRestoreSettings: SavepointRestoreSettings = {
savePoint match {
@@ -105,7 +116,10 @@ case class SubmitRequest(flinkVersion: FlinkVersion,
val format = this.appConf.substring(0, 7)
if (format == "json://") {
val json = this.appConf.drop(7)
- new ObjectMapper().readValue[JavaMap[String, String]](json, classOf[JavaMap[String, String]]).toMap.filter(_._2 != null)
+ new ObjectMapper()
+ .readValue[JavaMap[String, String]](json, classOf[JavaMap[String, String]])
+ .toMap
+ .filter(_._2 != null)
} else {
lazy val content = DeflaterUtils.unzipString(this.appConf.trim.drop(7))
val map = format match {
@@ -120,21 +134,23 @@ case class SubmitRequest(flinkVersion: FlinkVersion,
case "yml" | "yaml" => PropertiesUtils.fromYamlText(text)
case "conf" => PropertiesUtils.fromHoconText(text)
case "properties" => PropertiesUtils.fromPropertiesText(text)
- case _ => throw new IllegalArgumentException("[StreamPark] Usage: application config format error,must be [yaml|conf|properties]")
+ case _ =>
+ throw new IllegalArgumentException(
+ "[StreamPark] Usage: application config format error,must be [yaml|conf|properties]")
}
- case _ => throw new IllegalArgumentException("[StreamPark] application config format error.")
+ case _ =>
+ throw new IllegalArgumentException("[StreamPark] application config format error.")
}
map
- .filter(_._1.startsWith(prefix)).filter(_._2.nonEmpty)
+ .filter(_._1.startsWith(prefix))
+ .filter(_._2.nonEmpty)
.map(x => x._1.drop(prefix.length) -> x._2)
}
}
private[client] lazy val hdfsWorkspace = {
- /**
- * 必须保持本机flink和hdfs里的flink版本和配置都完全一致.
- */
+ /** 必须保持本机flink和hdfs里的flink版本和配置都完全一致. */
val workspace = Workspace.remote
val flinkHome = flinkVersion.flinkHome
val flinkHomeDir = new File(flinkHome)
@@ -151,7 +167,8 @@ case class SubmitRequest(flinkVersion: FlinkVersion,
flinkPlugins = s"$flinkHdfsHome/plugins",
flinkDistJar = FlinkUtils.getFlinkDistJar(flinkHome),
appJars = workspace.APP_JARS,
- appPlugins = workspace.APP_PLUGINS)
+ appPlugins = workspace.APP_PLUGINS
+ )
}
@throws[Exception]
@@ -159,21 +176,25 @@ case class SubmitRequest(flinkVersion: FlinkVersion,
executionMode match {
case ExecutionMode.KUBERNETES_NATIVE_SESSION | ExecutionMode.KUBERNETES_NATIVE_APPLICATION =>
if (buildResult == null) {
- throw new Exception(s"[flink-submit] current job: ${this.effectiveAppName} was not yet built, buildResult is empty" +
- s",clusterId=${k8sSubmitParam.clusterId}," +
- s",namespace=${k8sSubmitParam.kubernetesNamespace}")
+ throw new Exception(
+ s"[flink-submit] current job: ${this.effectiveAppName} was not yet built, buildResult is empty" +
+ s",clusterId=${k8sSubmitParam.clusterId}," +
+ s",namespace=${k8sSubmitParam.kubernetesNamespace}")
}
if (!buildResult.pass) {
- throw new Exception(s"[flink-submit] current job ${this.effectiveAppName} build failed, clusterId" +
- s",clusterId=${k8sSubmitParam.clusterId}," +
- s",namespace=${k8sSubmitParam.kubernetesNamespace}")
+ throw new Exception(
+ s"[flink-submit] current job ${this.effectiveAppName} build failed, clusterId" +
+ s",clusterId=${k8sSubmitParam.clusterId}," +
+ s",namespace=${k8sSubmitParam.kubernetesNamespace}")
}
case _ =>
if (this.buildResult == null) {
- throw new Exception(s"[flink-submit] current job: ${this.effectiveAppName} was not yet built, buildResult is empty")
+ throw new Exception(
+ s"[flink-submit] current job: ${this.effectiveAppName} was not yet built, buildResult is empty")
}
if (!this.buildResult.pass) {
- throw new Exception(s"[flink-submit] current job ${this.effectiveAppName} build failed, please check")
+ throw new Exception(
+ s"[flink-submit] current job ${this.effectiveAppName} build failed, please check")
}
}
}
@@ -189,10 +210,11 @@ case class SubmitRequest(flinkVersion: FlinkVersion,
* @param appJars
* @param appPlugins
*/
-case class HdfsWorkspace(flinkName: String,
- flinkHome: String,
- flinkDistJar: String,
- flinkLib: String,
- flinkPlugins: String,
- appJars: String,
- appPlugins: String)
+case class HdfsWorkspace(
+ flinkName: String,
+ flinkHome: String,
+ flinkDistJar: String,
+ flinkLib: String,
+ flinkPlugins: String,
+ appJars: String,
+ appPlugins: String)
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SubmitResponse.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SubmitResponse.scala
index c44987753..d4d8f037f 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SubmitResponse.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/SubmitResponse.scala
@@ -17,7 +17,12 @@
package org.apache.streampark.flink.client.bean
-import java.util.{Map => JavaMap}
import javax.annotation.Nullable
-case class SubmitResponse(clusterId: String, flinkConfig: JavaMap[String, String], @Nullable jobId: String = "", @Nullable jobManagerUrl: String = "")
+import java.util.{Map => JavaMap}
+
+case class SubmitResponse(
+ clusterId: String,
+ flinkConfig: JavaMap[String, String],
+ @Nullable jobId: String = "",
+ @Nullable jobManagerUrl: String = "")
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/TriggerSavepointRequest.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/TriggerSavepointRequest.scala
index f47069b7b..8f6344834 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/TriggerSavepointRequest.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/scala/org/apache/streampark/flink/client/bean/TriggerSavepointRequest.scala
@@ -17,18 +17,20 @@
package org.apache.streampark.flink.client.bean
-import java.util.{Map => JavaMap}
-
import org.apache.streampark.common.conf.{FlinkVersion, K8sFlinkConfig}
import org.apache.streampark.common.enums.ExecutionMode
import javax.annotation.Nullable
+import java.util.{Map => JavaMap}
+
/** Trigger savepoint request. */
-case class TriggerSavepointRequest(flinkVersion: FlinkVersion,
- executionMode: ExecutionMode,
- @Nullable properties: JavaMap[String, Any],
- clusterId: String,
- jobId: String,
- savepointPath: String,
- override val kubernetesNamespace: String = K8sFlinkConfig.DEFAULT_KUBERNETES_NAMESPACE) extends SavepointRequestTrait
+case class TriggerSavepointRequest(
+ flinkVersion: FlinkVersion,
+ executionMode: ExecutionMode,
+ @Nullable properties: JavaMap[String, Any],
+ clusterId: String,
+ jobId: String,
+ savepointPath: String,
+ override val kubernetesNamespace: String = K8sFlinkConfig.DEFAULT_KUBERNETES_NAMESPACE)
+ extends SavepointRequestTrait
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/FlinkClientHandler.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/FlinkClientHandler.scala
index 6c3be50b0..e85ceb6ee 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/FlinkClientHandler.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/FlinkClientHandler.scala
@@ -38,21 +38,25 @@ object FlinkClientHandler {
def submit(request: SubmitRequest): SubmitResponse = {
clients.get(request.executionMode) match {
case Some(client) => client.submit(request)
- case _ => throw new UnsupportedOperationException(s"Unsupported ${request.executionMode} submit ")
+ case _ =>
+ throw new UnsupportedOperationException(s"Unsupported ${request.executionMode} submit ")
}
}
def cancel(request: CancelRequest): CancelResponse = {
clients.get(request.executionMode) match {
case Some(client) => client.cancel(request)
- case _ => throw new UnsupportedOperationException(s"Unsupported ${request.executionMode} cancel ")
+ case _ =>
+ throw new UnsupportedOperationException(s"Unsupported ${request.executionMode} cancel ")
}
}
def triggerSavepoint(request: TriggerSavepointRequest): SavepointResponse = {
clients.get(request.executionMode) match {
case Some(client) => client.triggerSavepoint(request)
- case _ => throw new UnsupportedOperationException(s"Unsupported ${request.executionMode} triggerSavepoint ")
+ case _ =>
+ throw new UnsupportedOperationException(
+ s"Unsupported ${request.executionMode} triggerSavepoint ")
}
}
@@ -60,7 +64,9 @@ object FlinkClientHandler {
request.executionMode match {
case YARN_SESSION => YarnSessionClient.deploy(request)
case KUBERNETES_NATIVE_SESSION => KubernetesNativeSessionClient.deploy(request)
- case _ => throw new UnsupportedOperationException(s"Unsupported ${request.executionMode} deploy cluster ")
+ case _ =>
+ throw new UnsupportedOperationException(
+ s"Unsupported ${request.executionMode} deploy cluster ")
}
}
@@ -68,7 +74,9 @@ object FlinkClientHandler {
request.executionMode match {
case YARN_SESSION => YarnSessionClient.shutdown(request)
case KUBERNETES_NATIVE_SESSION => KubernetesNativeSessionClient.shutdown(request)
- case _ => throw new UnsupportedOperationException(s"Unsupported ${request.executionMode} shutdown cluster ")
+ case _ =>
+ throw new UnsupportedOperationException(
+ s"Unsupported ${request.executionMode} shutdown cluster ")
}
}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.scala
index d291eb42e..be57adb41 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.scala
@@ -17,6 +17,12 @@
package org.apache.streampark.flink.client.impl
+import org.apache.streampark.common.enums.ExecutionMode
+import org.apache.streampark.common.util.Utils
+import org.apache.streampark.flink.client.`trait`.KubernetesNativeClientTrait
+import org.apache.streampark.flink.client.bean._
+import org.apache.streampark.flink.packer.pipeline.DockerImageBuildResponse
+
import com.google.common.collect.Lists
import org.apache.commons.lang3.StringUtils
import org.apache.flink.client.deployment.application.ApplicationConfiguration
@@ -25,24 +31,19 @@ import org.apache.flink.configuration.{Configuration, DeploymentOptions, Pipelin
import org.apache.flink.kubernetes.KubernetesClusterDescriptor
import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions
-import org.apache.streampark.common.enums.ExecutionMode
-import org.apache.streampark.common.util.Utils
-import org.apache.streampark.flink.packer.pipeline.DockerImageBuildResponse
-import org.apache.streampark.flink.client.`trait`.KubernetesNativeClientTrait
-import org.apache.streampark.flink.client.bean._
-
-/**
- * kubernetes native application mode submit
- */
+/** kubernetes native application mode submit */
object KubernetesNativeApplicationClient extends KubernetesNativeClientTrait {
@throws[Exception]
- override def doSubmit(submitRequest: SubmitRequest, flinkConfig: Configuration): SubmitResponse = {
+ override def doSubmit(
+ submitRequest: SubmitRequest,
+ flinkConfig: Configuration): SubmitResponse = {
// require parameters
require(
StringUtils.isNotBlank(submitRequest.k8sSubmitParam.clusterId),
- s"[flink-submit] submit flink job failed, clusterId is null, mode=${flinkConfig.get(DeploymentOptions.TARGET)}")
+ s"[flink-submit] submit flink job failed, clusterId is null, mode=${flinkConfig.get(DeploymentOptions.TARGET)}"
+ )
// check the last building result
submitRequest.checkBuildResult()
@@ -50,7 +51,9 @@ object KubernetesNativeApplicationClient extends KubernetesNativeClientTrait {
val buildResult = submitRequest.buildResult.asInstanceOf[DockerImageBuildResponse]
// add flink pipeline.jars configuration
- flinkConfig.safeSet(PipelineOptions.JARS, Lists.newArrayList(buildResult.dockerInnerMainJarPath))
+ flinkConfig.safeSet(
+ PipelineOptions.JARS,
+ Lists.newArrayList(buildResult.dockerInnerMainJarPath))
// add flink container image tag to flink configuration
flinkConfig.safeSet(KubernetesConfigOptions.CONTAINER_IMAGE, buildResult.flinkImageTag)
@@ -68,8 +71,13 @@ object KubernetesNativeApplicationClient extends KubernetesNativeClientTrait {
.getClusterClient
val clusterId = clusterClient.getClusterId
- val result = SubmitResponse(clusterId, flinkConfig.toMap, submitRequest.jobId, clusterClient.getWebInterfaceURL)
- logInfo(s"[flink-submit] flink job has been submitted. ${flinkConfIdentifierInfo(flinkConfig)}")
+ val result = SubmitResponse(
+ clusterId,
+ flinkConfig.toMap,
+ submitRequest.jobId,
+ clusterClient.getWebInterfaceURL)
+ logInfo(
+ s"[flink-submit] flink job has been submitted. ${flinkConfIdentifierInfo(flinkConfig)}")
result
} catch {
case e: Exception =>
@@ -80,12 +88,18 @@ object KubernetesNativeApplicationClient extends KubernetesNativeClientTrait {
}
}
- override def doCancel(cancelRequest: CancelRequest, flinkConfig: Configuration): CancelResponse = {
- flinkConfig.safeSet(DeploymentOptions.TARGET, ExecutionMode.KUBERNETES_NATIVE_APPLICATION.getName)
+ override def doCancel(
+ cancelRequest: CancelRequest,
+ flinkConfig: Configuration): CancelResponse = {
+ flinkConfig.safeSet(
+ DeploymentOptions.TARGET,
+ ExecutionMode.KUBERNETES_NATIVE_APPLICATION.getName)
super.doCancel(cancelRequest, flinkConfig)
}
- override def doTriggerSavepoint(request: TriggerSavepointRequest, flinkConf: Configuration): SavepointResponse = {
+ override def doTriggerSavepoint(
+ request: TriggerSavepointRequest,
+ flinkConf: Configuration): SavepointResponse = {
flinkConf.safeSet(DeploymentOptions.TARGET, ExecutionMode.KUBERNETES_NATIVE_APPLICATION.getName)
super.doTriggerSavepoint(request, flinkConf)
}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.scala
index fb429d94e..346a8f1c6 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.scala
@@ -17,11 +17,15 @@
package org.apache.streampark.flink.client.impl
-import java.io.File
-
-import scala.collection.JavaConversions._
-import scala.language.postfixOps
-import scala.util.{Failure, Success, Try}
+import org.apache.streampark.common.enums.ExecutionMode
+import org.apache.streampark.common.util.{Logger, Utils}
+import org.apache.streampark.flink.client.`trait`.KubernetesNativeClientTrait
+import org.apache.streampark.flink.client.bean._
+import org.apache.streampark.flink.client.tool.FlinkSessionSubmitHelper
+import org.apache.streampark.flink.core.FlinkKubernetesClient
+import org.apache.streampark.flink.kubernetes.KubernetesRetriever
+import org.apache.streampark.flink.kubernetes.enums.FlinkK8sExecuteMode
+import org.apache.streampark.flink.kubernetes.model.ClusterKey
import io.fabric8.kubernetes.api.model.{Config => _}
import org.apache.commons.lang3.StringUtils
@@ -32,43 +36,44 @@ import org.apache.flink.kubernetes.configuration.{KubernetesConfigOptions, Kuber
import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions.ServiceExposedType
import org.apache.flink.kubernetes.kubeclient.{FlinkKubeClient, FlinkKubeClientFactory}
-import org.apache.streampark.common.enums.ExecutionMode
-import org.apache.streampark.common.util.{Logger, Utils}
-import org.apache.streampark.flink.core.FlinkKubernetesClient
-import org.apache.streampark.flink.kubernetes.KubernetesRetriever
-import org.apache.streampark.flink.kubernetes.enums.FlinkK8sExecuteMode
-import org.apache.streampark.flink.kubernetes.model.ClusterKey
-import org.apache.streampark.flink.client.`trait`.KubernetesNativeClientTrait
-import org.apache.streampark.flink.client.bean._
-import org.apache.streampark.flink.client.tool.FlinkSessionSubmitHelper
+import java.io.File
-/**
- * kubernetes native session mode submit
- */
+import scala.collection.JavaConversions._
+import scala.language.postfixOps
+import scala.util.{Failure, Success, Try}
+
+/** kubernetes native session mode submit */
object KubernetesNativeSessionClient extends KubernetesNativeClientTrait with Logger {
@throws[Exception]
- override def doSubmit(submitRequest: SubmitRequest, flinkConfig: Configuration): SubmitResponse = {
+ override def doSubmit(
+ submitRequest: SubmitRequest,
+ flinkConfig: Configuration): SubmitResponse = {
// require parameters
require(
StringUtils.isNotBlank(submitRequest.k8sSubmitParam.clusterId),
- s"[flink-submit] submit flink job failed, clusterId is null, mode=${flinkConfig.get(DeploymentOptions.TARGET)}")
- super.trySubmit(submitRequest, flinkConfig, submitRequest.userJarFile)(restApiSubmit)(jobGraphSubmit)
+ s"[flink-submit] submit flink job failed, clusterId is null, mode=${flinkConfig.get(DeploymentOptions.TARGET)}"
+ )
+ super.trySubmit(submitRequest, flinkConfig, submitRequest.userJarFile)(restApiSubmit)(
+ jobGraphSubmit)
}
- /**
- * Submit flink session job via rest api.
- */
+ /** Submit flink session job via rest api. */
@throws[Exception]
- def restApiSubmit(submitRequest: SubmitRequest, flinkConfig: Configuration, fatJar: File): SubmitResponse = {
+ def restApiSubmit(
+ submitRequest: SubmitRequest,
+ flinkConfig: Configuration,
+ fatJar: File): SubmitResponse = {
Try {
// get jm rest url of flink session cluster
val clusterKey = ClusterKey(
FlinkK8sExecuteMode.SESSION,
submitRequest.k8sSubmitParam.kubernetesNamespace,
submitRequest.k8sSubmitParam.clusterId)
- val jmRestUrl = KubernetesRetriever.retrieveFlinkRestUrl(clusterKey)
- .getOrElse(throw new Exception(s"[flink-submit] retrieve flink session rest url failed, clusterKey=$clusterKey"))
+ val jmRestUrl = KubernetesRetriever
+ .retrieveFlinkRestUrl(clusterKey)
+ .getOrElse(throw new Exception(
+ s"[flink-submit] retrieve flink session rest url failed, clusterKey=$clusterKey"))
// submit job via rest api
val jobId = FlinkSessionSubmitHelper.submitViaRestApi(jmRestUrl, fatJar, flinkConfig)
SubmitResponse(clusterKey.clusterId, flinkConfig.toMap, jobId, jmRestUrl)
@@ -80,11 +85,12 @@ object KubernetesNativeSessionClient extends KubernetesNativeClientTrait with Lo
}
}
- /**
- * Submit flink session job with building JobGraph via ClusterClient api.
- */
+ /** Submit flink session job with building JobGraph via ClusterClient api. */
@throws[Exception]
- def jobGraphSubmit(submitRequest: SubmitRequest, flinkConfig: Configuration, jarFile: File): SubmitResponse = {
+ def jobGraphSubmit(
+ submitRequest: SubmitRequest,
+ flinkConfig: Configuration,
+ jarFile: File): SubmitResponse = {
// retrieve k8s cluster and submit flink job on session mode
var clusterDescriptor: KubernetesClusterDescriptor = null
var packageProgram: PackagedProgram = null
@@ -97,11 +103,15 @@ object KubernetesNativeSessionClient extends KubernetesNativeClientTrait with Lo
packageProgram = packageProgramJobGraph._1
val jobGraph = packageProgramJobGraph._2
// retrieve client and submit JobGraph
- client = clusterDescriptor.retrieve(flinkConfig.getString(KubernetesConfigOptions.CLUSTER_ID)).getClusterClient
+ client = clusterDescriptor
+ .retrieve(flinkConfig.getString(KubernetesConfigOptions.CLUSTER_ID))
+ .getClusterClient
val submitResult = client.submitJob(jobGraph)
val jobId = submitResult.get().toString
- val result = SubmitResponse(client.getClusterId, flinkConfig.toMap, jobId, client.getWebInterfaceURL)
- logInfo(s"[flink-submit] flink job has been submitted. ${flinkConfIdentifierInfo(flinkConfig)}, jobId: $jobId")
+ val result =
+ SubmitResponse(client.getClusterId, flinkConfig.toMap, jobId, client.getWebInterfaceURL)
+ logInfo(
+ s"[flink-submit] flink job has been submitted. ${flinkConfIdentifierInfo(flinkConfig)}, jobId: $jobId")
result
} catch {
case e: Exception =>
@@ -116,7 +126,9 @@ object KubernetesNativeSessionClient extends KubernetesNativeClientTrait with Lo
}
}
- override def doCancel(cancelRequest: CancelRequest, flinkConfig: Configuration): CancelResponse = {
+ override def doCancel(
+ cancelRequest: CancelRequest,
+ flinkConfig: Configuration): CancelResponse = {
flinkConfig.safeSet(DeploymentOptions.TARGET, ExecutionMode.KUBERNETES_NATIVE_SESSION.getName)
super.doCancel(cancelRequest, flinkConfig)
}
@@ -140,26 +152,42 @@ object KubernetesNativeSessionClient extends KubernetesNativeClientTrait with Lo
var client: ClusterClient[String] = null
var kubeClient: FlinkKubeClient = null
try {
- val flinkConfig = extractConfiguration(deployRequest.flinkVersion.flinkHome, deployRequest.properties)
+ val flinkConfig =
+ extractConfiguration(deployRequest.flinkVersion.flinkHome, deployRequest.properties)
flinkConfig
.safeSet(DeploymentOptions.TARGET, KubernetesDeploymentTarget.SESSION.getName)
- .safeSet(KubernetesConfigOptions.NAMESPACE, deployRequest.k8sDeployParam.kubernetesNamespace)
- .safeSet(KubernetesConfigOptions.KUBERNETES_SERVICE_ACCOUNT, deployRequest.k8sDeployParam.serviceAccount)
- .safeSet(KubernetesConfigOptions.REST_SERVICE_EXPOSED_TYPE, ServiceExposedType.valueOf(deployRequest.k8sDeployParam.flinkRestExposedType.getName))
+ .safeSet(
+ KubernetesConfigOptions.NAMESPACE,
+ deployRequest.k8sDeployParam.kubernetesNamespace)
+ .safeSet(
+ KubernetesConfigOptions.KUBERNETES_SERVICE_ACCOUNT,
+ deployRequest.k8sDeployParam.serviceAccount)
+ .safeSet(
+ KubernetesConfigOptions.REST_SERVICE_EXPOSED_TYPE,
+ ServiceExposedType.valueOf(deployRequest.k8sDeployParam.flinkRestExposedType.getName))
.safeSet(KubernetesConfigOptions.CLUSTER_ID, deployRequest.clusterId)
.safeSet(KubernetesConfigOptions.CONTAINER_IMAGE, deployRequest.k8sDeployParam.flinkImage)
- .safeSet(KubernetesConfigOptions.KUBE_CONFIG_FILE, getDefaultKubernetesConf(deployRequest.k8sDeployParam.kubeConf))
- .safeSet(DeploymentOptionsInternal.CONF_DIR, s"${deployRequest.flinkVersion.flinkHome}/conf")
+ .safeSet(
+ KubernetesConfigOptions.KUBE_CONFIG_FILE,
+ getDefaultKubernetesConf(deployRequest.k8sDeployParam.kubeConf))
+ .safeSet(
+ DeploymentOptionsInternal.CONF_DIR,
+ s"${deployRequest.flinkVersion.flinkHome}/conf")
val kubernetesClusterDescriptor = getK8sClusterDescriptorAndSpecification(flinkConfig)
clusterDescriptor = kubernetesClusterDescriptor._1
kubeClient = FlinkKubeClientFactory.getInstance.fromConfiguration(flinkConfig, "client")
val kubeClientWrapper = new FlinkKubernetesClient(kubeClient)
- if (deployRequest.clusterId != null && kubeClientWrapper.getService(deployRequest.clusterId).isPresent) {
+ if (
+ deployRequest.clusterId != null && kubeClientWrapper
+ .getService(deployRequest.clusterId)
+ .isPresent
+ ) {
client = clusterDescriptor.retrieve(deployRequest.clusterId).getClusterClient
} else {
- client = clusterDescriptor.deploySessionCluster(kubernetesClusterDescriptor._2).getClusterClient
+ client =
+ clusterDescriptor.deploySessionCluster(kubernetesClusterDescriptor._2).getClusterClient
}
if (client.getWebInterfaceURL != null) {
DeployResponse(client.getWebInterfaceURL, client.getClusterId)
@@ -180,25 +208,39 @@ object KubernetesNativeSessionClient extends KubernetesNativeClientTrait with Lo
var kubeClient: FlinkKubeClient = null
try {
val flinkConfig = getFlinkDefaultConfiguration(shutDownRequest.flinkVersion.flinkHome)
- shutDownRequest.properties.foreach(m =>
- m._2 match {
- case v if v != null => flinkConfig.setString(m._1, m._2.toString)
- case _ =>
- })
+ shutDownRequest.properties.foreach(
+ m =>
+ m._2 match {
+ case v if v != null => flinkConfig.setString(m._1, m._2.toString)
+ case _ =>
+ })
flinkConfig
.safeSet(DeploymentOptions.TARGET, KubernetesDeploymentTarget.SESSION.getName)
- .safeSet(KubernetesConfigOptions.NAMESPACE, shutDownRequest.kubernetesDeployParam.kubernetesNamespace)
- .safeSet(KubernetesConfigOptions.KUBERNETES_SERVICE_ACCOUNT, shutDownRequest.kubernetesDeployParam.serviceAccount)
+ .safeSet(
+ KubernetesConfigOptions.NAMESPACE,
+ shutDownRequest.kubernetesDeployParam.kubernetesNamespace)
+ .safeSet(
+ KubernetesConfigOptions.KUBERNETES_SERVICE_ACCOUNT,
+ shutDownRequest.kubernetesDeployParam.serviceAccount)
.safeSet(
KubernetesConfigOptions.REST_SERVICE_EXPOSED_TYPE,
- ServiceExposedType.valueOf(shutDownRequest.kubernetesDeployParam.flinkRestExposedType.getName))
+ ServiceExposedType.valueOf(
+ shutDownRequest.kubernetesDeployParam.flinkRestExposedType.getName))
.safeSet(KubernetesConfigOptions.CLUSTER_ID, shutDownRequest.clusterId)
- .safeSet(KubernetesConfigOptions.CONTAINER_IMAGE, shutDownRequest.kubernetesDeployParam.flinkImage)
- .safeSet(KubernetesConfigOptions.KUBE_CONFIG_FILE, getDefaultKubernetesConf(shutDownRequest.kubernetesDeployParam.kubeConf))
+ .safeSet(
+ KubernetesConfigOptions.CONTAINER_IMAGE,
+ shutDownRequest.kubernetesDeployParam.flinkImage)
+ .safeSet(
+ KubernetesConfigOptions.KUBE_CONFIG_FILE,
+ getDefaultKubernetesConf(shutDownRequest.kubernetesDeployParam.kubeConf))
kubeClient = FlinkKubeClientFactory.getInstance.fromConfiguration(flinkConfig, "client")
val kubeClientWrapper = new FlinkKubernetesClient(kubeClient)
- if (shutDownRequest.clusterId != null && kubeClientWrapper.getService(shutDownRequest.clusterId).isPresent) {
+ if (
+ shutDownRequest.clusterId != null && kubeClientWrapper
+ .getService(shutDownRequest.clusterId)
+ .isPresent
+ ) {
kubeClient.stopAndCleanupCluster(shutDownRequest.clusterId)
ShutDownResponse()
} else {
@@ -214,7 +256,9 @@ object KubernetesNativeSessionClient extends KubernetesNativeClientTrait with Lo
}
}
- override def doTriggerSavepoint(request: TriggerSavepointRequest, flinkConfig: Configuration): SavepointResponse = {
+ override def doTriggerSavepoint(
+ request: TriggerSavepointRequest,
+ flinkConfig: Configuration): SavepointResponse = {
flinkConfig.safeSet(DeploymentOptions.TARGET, ExecutionMode.KUBERNETES_NATIVE_SESSION.getName)
super.doTriggerSavepoint(request, flinkConfig)
}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/LocalClient.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/LocalClient.scala
index 93c0d9089..380832fab 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/LocalClient.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/LocalClient.scala
@@ -17,7 +17,9 @@
package org.apache.streampark.flink.client.impl
-import java.lang.{Integer => JavaInt}
+import org.apache.streampark.common.util.Utils
+import org.apache.streampark.flink.client.`trait`.FlinkClientTrait
+import org.apache.streampark.flink.client.bean._
import org.apache.flink.client.deployment.executors.RemoteExecutor
import org.apache.flink.client.program.{ClusterClient, MiniClusterClient, PackagedProgram}
@@ -25,28 +27,28 @@ import org.apache.flink.client.program.MiniClusterClient.MiniClusterId
import org.apache.flink.configuration._
import org.apache.flink.runtime.minicluster.{MiniCluster, MiniClusterConfiguration}
-import org.apache.streampark.common.util.Utils
-import org.apache.streampark.flink.client.`trait`.FlinkClientTrait
-import org.apache.streampark.flink.client.bean._
+import java.lang.{Integer => JavaInt}
object LocalClient extends FlinkClientTrait {
override def setConfig(submitRequest: SubmitRequest, flinkConfig: Configuration): Unit = {
flinkConfig.safeSet(PipelineOptions.NAME, submitRequest.effectiveAppName)
- logInfo(
- s"""
- |------------------------------------------------------------------
- |Effective submit configuration: $flinkConfig
- |------------------------------------------------------------------
- |""".stripMargin)
+ logInfo(s"""
+ |------------------------------------------------------------------
+ |Effective submit configuration: $flinkConfig
+ |------------------------------------------------------------------
+ |""".stripMargin)
}
- override def doSubmit(submitRequest: SubmitRequest, flinkConfig: Configuration): SubmitResponse = {
+ override def doSubmit(
+ submitRequest: SubmitRequest,
+ flinkConfig: Configuration): SubmitResponse = {
var packageProgram: PackagedProgram = null
var client: ClusterClient[MiniClusterId] = null
try {
// build JobGraph
- val packageProgramJobGraph = super.getJobGraph(flinkConfig, submitRequest, submitRequest.userJarFile)
+ val packageProgramJobGraph =
+ super.getJobGraph(flinkConfig, submitRequest, submitRequest.userJarFile)
packageProgram = packageProgramJobGraph._1
val jobGraph = packageProgramJobGraph._2
client = createLocalCluster(flinkConfig)
@@ -65,11 +67,15 @@ object LocalClient extends FlinkClientTrait {
}
}
- override def doTriggerSavepoint(request: TriggerSavepointRequest, flinkConfig: Configuration): SavepointResponse = {
+ override def doTriggerSavepoint(
+ request: TriggerSavepointRequest,
+ flinkConfig: Configuration): SavepointResponse = {
RemoteClient.doTriggerSavepoint(request, flinkConfig)
}
- override def doCancel(cancelRequest: CancelRequest, flinkConfig: Configuration): CancelResponse = {
+ override def doCancel(
+ cancelRequest: CancelRequest,
+ flinkConfig: Configuration): CancelResponse = {
RemoteClient.doCancel(cancelRequest, flinkConfig)
}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/RemoteClient.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/RemoteClient.scala
index e8e614582..a696e9c7f 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/RemoteClient.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/RemoteClient.scala
@@ -17,24 +17,22 @@
package org.apache.streampark.flink.client.impl
-import java.io.File
-import java.lang.{Integer => JavaInt}
-
-import scala.util.{Failure, Success, Try}
+import org.apache.streampark.common.util.Utils
+import org.apache.streampark.flink.client.`trait`.FlinkClientTrait
+import org.apache.streampark.flink.client.bean.{CancelRequest, CancelResponse, SavepointRequestTrait, SavepointResponse, SubmitRequest, SubmitResponse, TriggerSavepointRequest}
+import org.apache.streampark.flink.client.tool.FlinkSessionSubmitHelper
import org.apache.flink.api.common.JobID
import org.apache.flink.client.deployment.{DefaultClusterClientServiceLoader, StandaloneClusterDescriptor, StandaloneClusterId}
import org.apache.flink.client.program.{ClusterClient, PackagedProgram}
import org.apache.flink.configuration._
-import org.apache.streampark.common.util.Utils
-import org.apache.streampark.flink.client.`trait`.FlinkClientTrait
-import org.apache.streampark.flink.client.bean.{CancelRequest, CancelResponse, TriggerSavepointRequest, SavepointRequestTrait, SavepointResponse, SubmitRequest, SubmitResponse}
-import org.apache.streampark.flink.client.tool.FlinkSessionSubmitHelper
+import java.io.File
+import java.lang.{Integer => JavaInt}
-/**
- * Submit Job to Remote Cluster
- */
+import scala.util.{Failure, Success, Try}
+
+/** Submit Job to Remote Cluster */
object RemoteClient extends FlinkClientTrait {
/**
@@ -43,34 +41,42 @@ object RemoteClient extends FlinkClientTrait {
*/
override def setConfig(submitRequest: SubmitRequest, flinkConfig: Configuration): Unit = {}
- override def doSubmit(submitRequest: SubmitRequest, flinkConfig: Configuration): SubmitResponse = {
+ override def doSubmit(
+ submitRequest: SubmitRequest,
+ flinkConfig: Configuration): SubmitResponse = {
// 2) submit job
- super.trySubmit(submitRequest, flinkConfig, submitRequest.userJarFile)(restApiSubmit)(jobGraphSubmit)
+ super.trySubmit(submitRequest, flinkConfig, submitRequest.userJarFile)(restApiSubmit)(
+ jobGraphSubmit)
}
override def doCancel(request: CancelRequest, flinkConfig: Configuration): CancelResponse = {
- executeClientAction(request, flinkConfig, (jobID, clusterClient) => {
- CancelResponse(super.cancelJob(request, jobID, clusterClient))
- })
+ executeClientAction(
+ request,
+ flinkConfig,
+ (jobID, clusterClient) => {
+ CancelResponse(super.cancelJob(request, jobID, clusterClient))
+ })
}
- private[this] def executeClientAction[O, R <: SavepointRequestTrait](request: R,
- flinkConfig: Configuration,
- actFunc: (JobID, ClusterClient[_]) => O): O = {
+ private[this] def executeClientAction[O, R <: SavepointRequestTrait](
+ request: R,
+ flinkConfig: Configuration,
+ actFunc: (JobID, ClusterClient[_]) => O): O = {
var client: ClusterClient[StandaloneClusterId] = null
var standAloneDescriptor: (StandaloneClusterId, StandaloneClusterDescriptor) = null
try {
flinkConfig
.safeSet(DeploymentOptions.TARGET, request.executionMode.getName)
.safeSet(RestOptions.ADDRESS, request.properties.get(RestOptions.ADDRESS.key()).toString)
- .safeSet[JavaInt](RestOptions.PORT, request.properties.get(RestOptions.PORT.key()).toString.toInt)
- logInfo(
- s"""
- |------------------------------------------------------------------
- |Effective submit configuration: $flinkConfig
- |------------------------------------------------------------------
- |""".stripMargin)
+ .safeSet[JavaInt](
+ RestOptions.PORT,
+ request.properties.get(RestOptions.PORT.key()).toString.toInt)
+ logInfo(s"""
+ |------------------------------------------------------------------
+ |Effective submit configuration: $flinkConfig
+ |------------------------------------------------------------------
+ |""".stripMargin)
standAloneDescriptor = getStandAloneClusterDescriptor(flinkConfig)
client = standAloneDescriptor._2.retrieve(standAloneDescriptor._1).getClusterClient
actFunc(JobID.fromHexString(request.jobId), client)
@@ -85,18 +91,24 @@ object RemoteClient extends FlinkClientTrait {
}
}
- override def doTriggerSavepoint(request: TriggerSavepointRequest, flinkConfig: Configuration): SavepointResponse = {
- executeClientAction(request, flinkConfig, (jobID, clusterClient) => {
- SavepointResponse(super.triggerSavepoint(request, jobID, clusterClient))
- })
+ override def doTriggerSavepoint(
+ request: TriggerSavepointRequest,
+ flinkConfig: Configuration): SavepointResponse = {
+ executeClientAction(
+ request,
+ flinkConfig,
+ (jobID, clusterClient) => {
+ SavepointResponse(super.triggerSavepoint(request, jobID, clusterClient))
+ })
}
- /**
- * Submit flink session job via rest api.
- */
+ /** Submit flink session job via rest api. */
// noinspection DuplicatedCode
@throws[Exception]
- def restApiSubmit(submitRequest: SubmitRequest, flinkConfig: Configuration, fatJar: File): SubmitResponse = {
+ def restApiSubmit(
+ submitRequest: SubmitRequest,
+ flinkConfig: Configuration,
+ fatJar: File): SubmitResponse = {
// retrieve standalone session cluster and submit flink job on session mode
var clusterDescriptor: StandaloneClusterDescriptor = null;
var client: ClusterClient[StandaloneClusterId] = null
@@ -106,8 +118,10 @@ object RemoteClient extends FlinkClientTrait {
clusterDescriptor = standAloneDescriptor._2
client = clusterDescriptor.retrieve(yarnClusterId).getClusterClient
- val jobId = FlinkSessionSubmitHelper.submitViaRestApi(client.getWebInterfaceURL, fatJar, flinkConfig)
- logInfo(s"${submitRequest.executionMode} mode submit by restApi, WebInterfaceURL ${client.getWebInterfaceURL}, jobId: $jobId")
+ val jobId =
+ FlinkSessionSubmitHelper.submitViaRestApi(client.getWebInterfaceURL, fatJar, flinkConfig)
+ logInfo(
+ s"${submitRequest.executionMode} mode submit by restApi, WebInterfaceURL ${client.getWebInterfaceURL}, jobId: $jobId")
SubmitResponse(null, flinkConfig.toMap, jobId, client.getWebInterfaceURL)
} match {
case Success(s) => s
@@ -117,11 +131,12 @@ object RemoteClient extends FlinkClientTrait {
}
}
- /**
- * Submit flink session job with building JobGraph via Standalone ClusterClient api.
- */
+ /** Submit flink session job with building JobGraph via Standalone ClusterClient api. */
@throws[Exception]
- def jobGraphSubmit(submitRequest: SubmitRequest, flinkConfig: Configuration, jarFile: File): SubmitResponse = {
+ def jobGraphSubmit(
+ submitRequest: SubmitRequest,
+ flinkConfig: Configuration,
+ jarFile: File): SubmitResponse = {
var clusterDescriptor: StandaloneClusterDescriptor = null;
var packageProgram: PackagedProgram = null
var client: ClusterClient[StandaloneClusterId] = null
@@ -134,7 +149,8 @@ object RemoteClient extends FlinkClientTrait {
val jobGraph = packageProgramJobGraph._2
client = clusterDescriptor.retrieve(standAloneDescriptor._1).getClusterClient
val jobId = client.submitJob(jobGraph).get().toString
- logInfo(s"${submitRequest.executionMode} mode submit by jobGraph, WebInterfaceURL ${client.getWebInterfaceURL}, jobId: $jobId")
+ logInfo(
+ s"${submitRequest.executionMode} mode submit by jobGraph, WebInterfaceURL ${client.getWebInterfaceURL}, jobId: $jobId")
val result = SubmitResponse(null, flinkConfig.toMap, jobId, client.getWebInterfaceURL)
result
} catch {
@@ -155,11 +171,13 @@ object RemoteClient extends FlinkClientTrait {
*
* @param flinkConfig
*/
- private[this] def getStandAloneClusterDescriptor(flinkConfig: Configuration): (StandaloneClusterId, StandaloneClusterDescriptor) = {
+ private[this] def getStandAloneClusterDescriptor(
+ flinkConfig: Configuration): (StandaloneClusterId, StandaloneClusterDescriptor) = {
val serviceLoader = new DefaultClusterClientServiceLoader
val clientFactory = serviceLoader.getClusterClientFactory(flinkConfig)
val standaloneClusterId: StandaloneClusterId = clientFactory.getClusterId(flinkConfig)
- val standaloneClusterDescriptor = clientFactory.createClusterDescriptor(flinkConfig).asInstanceOf[StandaloneClusterDescriptor]
+ val standaloneClusterDescriptor =
+ clientFactory.createClusterDescriptor(flinkConfig).asInstanceOf[StandaloneClusterDescriptor]
(standaloneClusterId, standaloneClusterDescriptor)
}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnApplicationClient.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnApplicationClient.scala
index a4d6015b1..2b981b305 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnApplicationClient.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnApplicationClient.scala
@@ -17,11 +17,12 @@
package org.apache.streampark.flink.client.impl
-import java.util.Collections
-import java.util.concurrent.Callable
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ListBuffer
+import org.apache.streampark.common.conf.Workspace
+import org.apache.streampark.common.enums.DevelopmentMode
+import org.apache.streampark.common.util.{HdfsUtils, Utils}
+import org.apache.streampark.flink.client.`trait`.YarnClientTrait
+import org.apache.streampark.flink.client.bean._
+import org.apache.streampark.flink.packer.pipeline.ShadedBuildResponse
import org.apache.flink.client.deployment.DefaultClusterClientServiceLoader
import org.apache.flink.client.deployment.application.ApplicationConfiguration
@@ -33,30 +34,29 @@ import org.apache.flink.yarn.configuration.YarnConfigOptions
import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.yarn.api.records.ApplicationId
-import org.apache.streampark.common.conf.Workspace
-import org.apache.streampark.common.enums.DevelopmentMode
-import org.apache.streampark.common.util.{HdfsUtils, Utils}
-import org.apache.streampark.flink.packer.pipeline.ShadedBuildResponse
-import org.apache.streampark.flink.client.`trait`.YarnClientTrait
-import org.apache.streampark.flink.client.bean._
+import java.util.Collections
+import java.util.concurrent.Callable
-/**
- * yarn application mode submit
- */
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ListBuffer
+
+/** yarn application mode submit */
object YarnApplicationClient extends YarnClientTrait {
private[this] lazy val workspace = Workspace.remote
override def setConfig(submitRequest: SubmitRequest, flinkConfig: Configuration): Unit = {
- val flinkDefaultConfiguration = getFlinkDefaultConfiguration(submitRequest.flinkVersion.flinkHome)
+ val flinkDefaultConfiguration = getFlinkDefaultConfiguration(
+ submitRequest.flinkVersion.flinkHome)
val currentUser = UserGroupInformation.getCurrentUser
logDebug(s"UserGroupInformation currentUser: $currentUser")
if (HadoopUtils.isKerberosSecurityEnabled(currentUser)) {
logDebug(s"kerberos Security is Enabled...")
- val useTicketCache = flinkDefaultConfiguration.get(SecurityOptions.KERBEROS_LOGIN_USETICKETCACHE)
+ val useTicketCache =
+ flinkDefaultConfiguration.get(SecurityOptions.KERBEROS_LOGIN_USETICKETCACHE)
if (!HadoopUtils.areKerberosCredentialsValid(currentUser, useTicketCache)) {
throw new RuntimeException(
- s"Hadoop security with Kerberos is enabled but the login user ${currentUser} does not have Kerberos credentials or delegation tokens!")
+ s"Hadoop security with Kerberos is enabled but the login user $currentUser does not have Kerberos credentials or delegation tokens!")
}
}
val providedLibs = {
@@ -64,7 +64,8 @@ object YarnApplicationClient extends YarnClientTrait {
submitRequest.hdfsWorkspace.flinkLib,
submitRequest.hdfsWorkspace.flinkPlugins,
submitRequest.hdfsWorkspace.appJars,
- submitRequest.hdfsWorkspace.appPlugins)
+ submitRequest.hdfsWorkspace.appPlugins
+ )
submitRequest.developmentMode match {
case DevelopmentMode.FLINK_SQL =>
array += s"${workspace.APP_SHIMS}/flink-${submitRequest.flinkVersion.majorVersion}"
@@ -83,49 +84,54 @@ object YarnApplicationClient extends YarnClientTrait {
// flinkDistJar
.safeSet(YarnConfigOptions.FLINK_DIST_JAR, submitRequest.hdfsWorkspace.flinkDistJar)
// pipeline.jars
- .safeSet(PipelineOptions.JARS, Collections.singletonList(submitRequest.buildResult.asInstanceOf[ShadedBuildResponse].shadedJarPath))
+ .safeSet(
+ PipelineOptions.JARS,
+ Collections.singletonList(
+ submitRequest.buildResult.asInstanceOf[ShadedBuildResponse].shadedJarPath))
// yarn application name
.safeSet(YarnConfigOptions.APPLICATION_NAME, submitRequest.effectiveAppName)
// yarn application Type
.safeSet(YarnConfigOptions.APPLICATION_TYPE, submitRequest.applicationType.getName)
- logInfo(
- s"""
- |------------------------------------------------------------------
- |Effective submit configuration: $flinkConfig
- |------------------------------------------------------------------
- |""".stripMargin)
+ logInfo(s"""
+ |------------------------------------------------------------------
+ |Effective submit configuration: $flinkConfig
+ |------------------------------------------------------------------
+ |""".stripMargin)
}
- override def doSubmit(submitRequest: SubmitRequest, flinkConfig: Configuration): SubmitResponse = {
+ override def doSubmit(
+ submitRequest: SubmitRequest,
+ flinkConfig: Configuration): SubmitResponse = {
SecurityUtils.install(new SecurityConfiguration(flinkConfig))
SecurityUtils.getInstalledContext.runSecured(new Callable[SubmitResponse] {
override def call(): SubmitResponse = {
val clusterClientServiceLoader = new DefaultClusterClientServiceLoader
- val clientFactory = clusterClientServiceLoader.getClusterClientFactory[ApplicationId](flinkConfig)
+ val clientFactory =
+ clusterClientServiceLoader.getClusterClientFactory[ApplicationId](flinkConfig)
val clusterDescriptor = clientFactory.createClusterDescriptor(flinkConfig)
var clusterClient: ClusterClient[ApplicationId] = null
try {
val clusterSpecification = clientFactory.getClusterSpecification(flinkConfig)
- logInfo(
- s"""
- |------------------------<<specification>>-------------------------
- |$clusterSpecification
- |------------------------------------------------------------------
- |""".stripMargin)
+ logInfo(s"""
+ |------------------------<<specification>>-------------------------
+ |$clusterSpecification
+ |------------------------------------------------------------------
+ |""".stripMargin)
val applicationConfiguration = ApplicationConfiguration.fromConfiguration(flinkConfig)
var applicationId: ApplicationId = null
var jobManagerUrl: String = null
- clusterClient = clusterDescriptor.deployApplicationCluster(clusterSpecification, applicationConfiguration).getClusterClient
+ clusterClient = clusterDescriptor
+ .deployApplicationCluster(clusterSpecification, applicationConfiguration)
+ .getClusterClient
applicationId = clusterClient.getClusterId
jobManagerUrl = clusterClient.getWebInterfaceURL
- logInfo(
- s"""
- |-------------------------<<applicationId>>------------------------
- |Flink Job Started: applicationId: $applicationId
- |__________________________________________________________________
- |""".stripMargin)
+ logInfo(s"""
+ |-------------------------<<applicationId>>------------------------
+ |Flink Job Started: applicationId: $applicationId
+ |__________________________________________________________________
+ |""".stripMargin)
SubmitResponse(applicationId.toString, flinkConfig.toMap, jobManagerUrl = jobManagerUrl)
} finally {
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnPerJobClient.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnPerJobClient.scala
index 3a7990fa0..79813adf2 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnPerJobClient.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnPerJobClient.scala
@@ -17,10 +17,10 @@
package org.apache.streampark.flink.client.impl
-import java.io.File
-import java.lang.{Boolean => JavaBool}
-
-import scala.collection.JavaConversions._
+import org.apache.streampark.common.util.Utils
+import org.apache.streampark.flink.client.`trait`.YarnClientTrait
+import org.apache.streampark.flink.client.bean._
+import org.apache.streampark.flink.util.FlinkUtils
import org.apache.flink.client.deployment.DefaultClusterClientServiceLoader
import org.apache.flink.client.program.{ClusterClient, PackagedProgram}
@@ -31,14 +31,12 @@ import org.apache.flink.yarn.entrypoint.YarnJobClusterEntrypoint
import org.apache.hadoop.fs.{Path => HadoopPath}
import org.apache.hadoop.yarn.api.records.ApplicationId
-import org.apache.streampark.common.util.Utils
-import org.apache.streampark.flink.client.`trait`.YarnClientTrait
-import org.apache.streampark.flink.client.bean._
-import org.apache.streampark.flink.util.FlinkUtils
+import java.io.File
+import java.lang.{Boolean => JavaBool}
-/**
- * yarn PerJob mode submit
- */
+import scala.collection.JavaConversions._
+
+/** yarn PerJob mode submit */
object YarnPerJobClient extends YarnClientTrait {
override def setConfig(submitRequest: SubmitRequest, flinkConfig: Configuration): Unit = {
@@ -48,25 +46,28 @@ object YarnPerJobClient extends YarnClientTrait {
.safeSet(DeploymentOptions.ATTACHED, JavaBool.TRUE)
.safeSet(DeploymentOptions.SHUTDOWN_IF_ATTACHED, JavaBool.TRUE)
- logInfo(
- s"""
- |------------------------------------------------------------------
- |Effective submit configuration: $flinkConfig
- |------------------------------------------------------------------
- |""".stripMargin)
+ logInfo(s"""
+ |------------------------------------------------------------------
+ |Effective submit configuration: $flinkConfig
+ |------------------------------------------------------------------
+ |""".stripMargin)
}
- override def doSubmit(submitRequest: SubmitRequest, flinkConfig: Configuration): SubmitResponse = {
+ override def doSubmit(
+ submitRequest: SubmitRequest,
+ flinkConfig: Configuration): SubmitResponse = {
val flinkHome = submitRequest.flinkVersion.flinkHome
val clusterClientServiceLoader = new DefaultClusterClientServiceLoader
- val clientFactory = clusterClientServiceLoader.getClusterClientFactory[ApplicationId](flinkConfig)
+ val clientFactory =
+ clusterClientServiceLoader.getClusterClientFactory[ApplicationId](flinkConfig)
var packagedProgram: PackagedProgram = null
var clusterClient: ClusterClient[ApplicationId] = null
val clusterDescriptor = {
- val clusterDescriptor = clientFactory.createClusterDescriptor(flinkConfig).asInstanceOf[YarnClusterDescriptor]
+ val clusterDescriptor =
+ clientFactory.createClusterDescriptor(flinkConfig).asInstanceOf[YarnClusterDescriptor]
val flinkDistJar = FlinkUtils.getFlinkDistJar(flinkHome)
clusterDescriptor.setLocalJarPath(new HadoopPath(flinkDistJar))
clusterDescriptor.addShipFiles(List(new File(s"$flinkHome/lib")))
@@ -76,23 +77,22 @@ object YarnPerJobClient extends YarnClientTrait {
try {
clusterClient = {
val clusterSpecification = clientFactory.getClusterSpecification(flinkConfig)
- logInfo(
- s"""
- |------------------------<<specification>>-------------------------
- |$clusterSpecification
- |------------------------------------------------------------------
- |""".stripMargin)
-
- val packageProgramJobGraph = super.getJobGraph(flinkConfig, submitRequest, submitRequest.userJarFile)
+ logInfo(s"""
+ |------------------------<<specification>>-------------------------
+ |$clusterSpecification
+ |------------------------------------------------------------------
+ |""".stripMargin)
+
+ val packageProgramJobGraph =
+ super.getJobGraph(flinkConfig, submitRequest, submitRequest.userJarFile)
packagedProgram = packageProgramJobGraph._1
val jobGraph = packageProgramJobGraph._2
- logInfo(
- s"""
- |-------------------------<<applicationId>>------------------------
- |jobGraph getJobID: ${jobGraph.getJobID.toString}
- |__________________________________________________________________
- |""".stripMargin)
+ logInfo(s"""
+ |-------------------------<<applicationId>>------------------------
+ |jobGraph getJobID: ${jobGraph.getJobID.toString}
+ |__________________________________________________________________
+ |""".stripMargin)
deployInternal(
clusterDescriptor,
clusterSpecification,
@@ -104,12 +104,11 @@ object YarnPerJobClient extends YarnClientTrait {
}
val applicationId = clusterClient.getClusterId
val jobManagerUrl = clusterClient.getWebInterfaceURL
- logInfo(
- s"""
- |-------------------------<<applicationId>>------------------------
- |Flink Job Started: applicationId: $applicationId
- |__________________________________________________________________
- |""".stripMargin)
+ logInfo(s"""
+ |-------------------------<<applicationId>>------------------------
+ |Flink Job Started: applicationId: $applicationId
+ |__________________________________________________________________
+ |""".stripMargin)
SubmitResponse(applicationId.toString, flinkConfig.toMap, jobManagerUrl = jobManagerUrl)
} finally {
@@ -120,7 +119,9 @@ object YarnPerJobClient extends YarnClientTrait {
}
}
- override def doCancel(cancelRequest: CancelRequest, flinkConfig: Configuration): CancelResponse = {
+ override def doCancel(
+ cancelRequest: CancelRequest,
+ flinkConfig: Configuration): CancelResponse = {
val response = super.doCancel(cancelRequest, flinkConfig)
val clusterClientFactory = new YarnClusterClientFactory
val clusterDescriptor = clusterClientFactory.createClusterDescriptor(flinkConfig)
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnSessionClient.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnSessionClient.scala
index 5426fc6ca..595e983ea 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnSessionClient.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/impl/YarnSessionClient.scala
@@ -17,9 +17,9 @@
package org.apache.streampark.flink.client.impl
-import scala.collection.JavaConversions._
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ListBuffer
+import org.apache.streampark.common.util.Utils
+import org.apache.streampark.flink.client.`trait`.YarnClientTrait
+import org.apache.streampark.flink.client.bean._
import org.apache.flink.api.common.JobID
import org.apache.flink.client.deployment.DefaultClusterClientServiceLoader
@@ -33,13 +33,11 @@ import org.apache.hadoop.yarn.api.records.{ApplicationId, FinalApplicationStatus
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException
import org.apache.hadoop.yarn.util.ConverterUtils
-import org.apache.streampark.common.util.Utils
-import org.apache.streampark.flink.client.`trait`.YarnClientTrait
-import org.apache.streampark.flink.client.bean._
+import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ListBuffer
-/**
- * Submit Job to YARN Session Cluster
- */
+/** Submit Job to YARN Session Cluster */
object YarnSessionClient extends YarnClientTrait {
/**
@@ -49,12 +47,11 @@ object YarnSessionClient extends YarnClientTrait {
override def setConfig(submitRequest: SubmitRequest, flinkConfig: Configuration): Unit = {
flinkConfig
.safeSet(DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName)
- logInfo(
- s"""
- |------------------------------------------------------------------
- |Effective submit configuration: $flinkConfig
- |------------------------------------------------------------------
- |""".stripMargin)
+ logInfo(s"""
+ |------------------------------------------------------------------
+ |Effective submit configuration: $flinkConfig
+ |------------------------------------------------------------------
+ |""".stripMargin)
}
/**
@@ -62,15 +59,17 @@ object YarnSessionClient extends YarnClientTrait {
* @param flinkConfig
*/
def setConfig(deployRequest: DeployRequest, flinkConfig: Configuration): Unit = {
- val flinkDefaultConfiguration = getFlinkDefaultConfiguration(deployRequest.flinkVersion.flinkHome)
+ val flinkDefaultConfiguration = getFlinkDefaultConfiguration(
+ deployRequest.flinkVersion.flinkHome)
val currentUser = UserGroupInformation.getCurrentUser
logDebug(s"UserGroupInformation currentUser: $currentUser")
if (HadoopUtils.isKerberosSecurityEnabled(currentUser)) {
logDebug(s"kerberos Security is Enabled...")
- val useTicketCache = flinkDefaultConfiguration.get(SecurityOptions.KERBEROS_LOGIN_USETICKETCACHE)
+ val useTicketCache =
+ flinkDefaultConfiguration.get(SecurityOptions.KERBEROS_LOGIN_USETICKETCACHE)
if (!HadoopUtils.areKerberosCredentialsValid(currentUser, useTicketCache)) {
throw new RuntimeException(
- s"Hadoop security with Kerberos is enabled but the login user ${currentUser} does not have Kerberos credentials or delegation tokens!")
+ s"Hadoop security with Kerberos is enabled but the login user $currentUser does not have Kerberos credentials or delegation tokens!")
}
}
val providedLibs = {
@@ -78,7 +77,8 @@ object YarnSessionClient extends YarnClientTrait {
deployRequest.hdfsWorkspace.flinkLib,
deployRequest.hdfsWorkspace.flinkPlugins,
deployRequest.hdfsWorkspace.appJars,
- deployRequest.hdfsWorkspace.appPlugins)
+ deployRequest.hdfsWorkspace.appPlugins
+ )
array.toList
}
@@ -92,15 +92,16 @@ object YarnSessionClient extends YarnClientTrait {
// conf dir
.safeSet(DeploymentOptionsInternal.CONF_DIR, s"${deployRequest.flinkVersion.flinkHome}/conf")
- logInfo(
- s"""
- |------------------------------------------------------------------
- |Effective submit configuration: $flinkConfig
- |------------------------------------------------------------------
- |""".stripMargin)
+ logInfo(s"""
+ |------------------------------------------------------------------
+ |Effective submit configuration: $flinkConfig
+ |------------------------------------------------------------------
+ |""".stripMargin)
}
- override def doSubmit(submitRequest: SubmitRequest, flinkConfig: Configuration): SubmitResponse = {
+ override def doSubmit(
+ submitRequest: SubmitRequest,
+ flinkConfig: Configuration): SubmitResponse = {
var clusterDescriptor: YarnClusterDescriptor = null
var packageProgram: PackagedProgram = null
var client: ClusterClient[ApplicationId] = null
@@ -108,7 +109,8 @@ object YarnSessionClient extends YarnClientTrait {
val yarnClusterDescriptor = getYarnSessionClusterDescriptor(flinkConfig)
clusterDescriptor = yarnClusterDescriptor._2
val yarnClusterId: ApplicationId = yarnClusterDescriptor._1
- val packageProgramJobGraph = super.getJobGraph(flinkConfig, submitRequest, submitRequest.userJarFile)
+ val packageProgramJobGraph =
+ super.getJobGraph(flinkConfig, submitRequest, submitRequest.userJarFile)
packageProgram = packageProgramJobGraph._1
val jobGraph = packageProgramJobGraph._2
@@ -116,12 +118,11 @@ object YarnSessionClient extends YarnClientTrait {
val jobId = client.submitJob(jobGraph).get().toString
val jobManagerUrl = client.getWebInterfaceURL
- logInfo(
- s"""
- |-------------------------<<applicationId>>------------------------
- |Flink Job Started: jobId: $jobId , applicationId: ${yarnClusterId.toString}
- |__________________________________________________________________
- |""".stripMargin)
+ logInfo(s"""
+ |-------------------------<<applicationId>>------------------------
+ |Flink Job Started: jobId: $jobId , applicationId: ${yarnClusterId.toString}
+ |__________________________________________________________________
+ |""".stripMargin)
SubmitResponse(yarnClusterId.toString, flinkConfig.toMap, jobId, jobManagerUrl)
} catch {
case e: Exception =>
@@ -136,18 +137,18 @@ object YarnSessionClient extends YarnClientTrait {
}
}
- private[this] def executeClientAction[O, R <: SavepointRequestTrait](request: R,
- flinkConfig: Configuration,
- actFunc: (JobID, ClusterClient[_]) => O): O = {
+ private[this] def executeClientAction[O, R <: SavepointRequestTrait](
+ request: R,
+ flinkConfig: Configuration,
+ actFunc: (JobID, ClusterClient[_]) => O): O = {
flinkConfig
.safeSet(YarnConfigOptions.APPLICATION_ID, request.clusterId)
.safeSet(DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName)
- logInfo(
- s"""
- |------------------------------------------------------------------
- |Effective submit configuration: $flinkConfig
- |------------------------------------------------------------------
- |""".stripMargin)
+ logInfo(s"""
+ |------------------------------------------------------------------
+ |Effective submit configuration: $flinkConfig
+ |------------------------------------------------------------------
+ |""".stripMargin)
var clusterDescriptor: YarnClusterDescriptor = null
var client: ClusterClient[ApplicationId] = null
@@ -166,18 +167,28 @@ object YarnSessionClient extends YarnClientTrait {
}
}
- override def doCancel(cancelRequest: CancelRequest, flinkConfig: Configuration): CancelResponse = {
- executeClientAction(cancelRequest, flinkConfig, (jobID, clusterClient) => {
- val actionResult = super.cancelJob(cancelRequest, jobID, clusterClient)
- CancelResponse(actionResult)
- })
+ override def doCancel(
+ cancelRequest: CancelRequest,
+ flinkConfig: Configuration): CancelResponse = {
+ executeClientAction(
+ cancelRequest,
+ flinkConfig,
+ (jobID, clusterClient) => {
+ val actionResult = super.cancelJob(cancelRequest, jobID, clusterClient)
+ CancelResponse(actionResult)
+ })
}
- override def doTriggerSavepoint(request: TriggerSavepointRequest, flinkConfig: Configuration): SavepointResponse = {
- executeClientAction(request, flinkConfig, (jobID, clusterClient) => {
- val actionResult = super.triggerSavepoint(request, jobID, clusterClient)
- SavepointResponse(actionResult)
- })
+ override def doTriggerSavepoint(
+ request: TriggerSavepointRequest,
+ flinkConfig: Configuration): SavepointResponse = {
+ executeClientAction(
+ request,
+ flinkConfig,
+ (jobID, clusterClient) => {
+ val actionResult = super.triggerSavepoint(request, jobID, clusterClient)
+ SavepointResponse(actionResult)
+ })
}
def deploy(deployRequest: DeployRequest): DeployResponse = {
@@ -194,25 +205,29 @@ object YarnSessionClient extends YarnClientTrait {
var clusterDescriptor: YarnClusterDescriptor = null
var client: ClusterClient[ApplicationId] = null
try {
- val flinkConfig = extractConfiguration(
- deployRequest.flinkVersion.flinkHome,
- deployRequest.properties)
+ val flinkConfig =
+ extractConfiguration(deployRequest.flinkVersion.flinkHome, deployRequest.properties)
setConfig(deployRequest, flinkConfig)
val yarnClusterDescriptor = getSessionClusterDeployDescriptor(flinkConfig)
clusterDescriptor = yarnClusterDescriptor._2
if (null != deployRequest.clusterId && deployRequest.clusterId.nonEmpty) {
try {
val applicationStatus =
- clusterDescriptor.getYarnClient.getApplicationReport(ConverterUtils.toApplicationId(deployRequest.clusterId)).getFinalApplicationStatus
+ clusterDescriptor.getYarnClient
+ .getApplicationReport(ConverterUtils.toApplicationId(deployRequest.clusterId))
+ .getFinalApplicationStatus
if (FinalApplicationStatus.UNDEFINED.equals(applicationStatus)) {
// application is running
- val yarnClient = clusterDescriptor.retrieve(ApplicationId.fromString(deployRequest.clusterId)).getClusterClient
+ val yarnClient = clusterDescriptor
+ .retrieve(ApplicationId.fromString(deployRequest.clusterId))
+ .getClusterClient
if (yarnClient.getWebInterfaceURL != null) {
return DeployResponse(yarnClient.getWebInterfaceURL, yarnClient.getClusterId.toString)
}
}
} catch {
- case _: ApplicationNotFoundException => logInfo("this applicationId have not managed by yarn ,need deploy ...")
+ case _: ApplicationNotFoundException =>
+ logInfo("this applicationId have not managed by yarn ,need deploy ...")
}
}
val clientProvider = clusterDescriptor.deploySessionCluster(yarnClusterDescriptor._1)
@@ -237,23 +252,29 @@ object YarnSessionClient extends YarnClientTrait {
var client: ClusterClient[ApplicationId] = null
try {
val flinkConfig = getFlinkDefaultConfiguration(shutDownRequest.flinkVersion.flinkHome)
- shutDownRequest.properties.foreach(m =>
- m._2 match {
- case v if v != null => flinkConfig.setString(m._1, m._2.toString)
- case _ =>
- })
+ shutDownRequest.properties.foreach(
+ m =>
+ m._2 match {
+ case v if v != null => flinkConfig.setString(m._1, m._2.toString)
+ case _ =>
+ })
flinkConfig.safeSet(YarnConfigOptions.APPLICATION_ID, shutDownRequest.clusterId)
flinkConfig.safeSet(DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName)
val yarnClusterDescriptor = getSessionClusterDescriptor(flinkConfig)
clusterDescriptor = yarnClusterDescriptor._2
- if (FinalApplicationStatus.UNDEFINED.equals(
- clusterDescriptor.getYarnClient.getApplicationReport(ApplicationId.fromString(shutDownRequest.clusterId)).getFinalApplicationStatus)) {
+ if (
+ FinalApplicationStatus.UNDEFINED.equals(
+ clusterDescriptor.getYarnClient
+ .getApplicationReport(ApplicationId.fromString(shutDownRequest.clusterId))
+ .getFinalApplicationStatus)
+ ) {
val clientProvider = clusterDescriptor.retrieve(yarnClusterDescriptor._1)
client = clientProvider.getClusterClient
client.shutDownCluster()
}
- logInfo(s"the ${shutDownRequest.clusterId}'s final status is ${clusterDescriptor.getYarnClient.getApplicationReport(
- ConverterUtils.toApplicationId(shutDownRequest.clusterId)).getFinalApplicationStatus}")
+ logInfo(s"the ${shutDownRequest.clusterId}'s final status is ${clusterDescriptor.getYarnClient
+ .getApplicationReport(ConverterUtils.toApplicationId(shutDownRequest.clusterId))
+ .getFinalApplicationStatus}")
ShutDownResponse()
} catch {
case e: Exception =>
@@ -265,12 +286,14 @@ object YarnSessionClient extends YarnClientTrait {
}
}
- private[this] def getYarnSessionClusterDescriptor(flinkConfig: Configuration): (ApplicationId, YarnClusterDescriptor) = {
+ private[this] def getYarnSessionClusterDescriptor(
+ flinkConfig: Configuration): (ApplicationId, YarnClusterDescriptor) = {
val serviceLoader = new DefaultClusterClientServiceLoader
val clientFactory = serviceLoader.getClusterClientFactory[ApplicationId](flinkConfig)
val yarnClusterId: ApplicationId = clientFactory.getClusterId(flinkConfig)
require(yarnClusterId != null)
- val clusterDescriptor = clientFactory.createClusterDescriptor(flinkConfig).asInstanceOf[YarnClusterDescriptor]
+ val clusterDescriptor =
+ clientFactory.createClusterDescriptor(flinkConfig).asInstanceOf[YarnClusterDescriptor]
(yarnClusterId, clusterDescriptor)
}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/tool/FlinkSessionClientHelper.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/tool/FlinkSessionClientHelper.scala
index a9f9ba039..e7d673628 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/tool/FlinkSessionClientHelper.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/tool/FlinkSessionClientHelper.scala
@@ -17,10 +17,8 @@
package org.apache.streampark.flink.client.tool
-import java.io.File
-import java.nio.charset.StandardCharsets
-
-import scala.util.{Failure, Success, Try}
+import org.apache.streampark.common.util.Logger
+import org.apache.streampark.flink.kubernetes.KubernetesRetriever
import org.apache.flink.client.deployment.application.ApplicationConfiguration
import org.apache.flink.configuration.{Configuration, CoreOptions}
@@ -34,8 +32,10 @@ import org.json4s.DefaultFormats
import org.json4s.jackson.JsonMethods._
import org.json4s.jackson.Serialization
-import org.apache.streampark.common.util.Logger
-import org.apache.streampark.flink.kubernetes.KubernetesRetriever
+import java.io.File
+import java.nio.charset.StandardCharsets
+
+import scala.util.{Failure, Success, Try}
object FlinkSessionSubmitHelper extends Logger {
@@ -45,21 +45,32 @@ object FlinkSessionSubmitHelper extends Logger {
/**
* Submit Flink Job via Rest API.
*
- * @param jmRestUrl jobmanager rest url of target flink cluster
- * @param flinkJobJar flink job jar file
- * @param flinkConfig flink configuration
- * @return jobID of submitted flink job
+ * @param jmRestUrl
+ * jobmanager rest url of target flink cluster
+ * @param flinkJobJar
+ * flink job jar file
+ * @param flinkConfig
+ * flink configuration
+ * @return
+ * jobID of submitted flink job
*/
@throws[Exception]
def submitViaRestApi(jmRestUrl: String, flinkJobJar: File, flinkConfig: Configuration): String = {
// upload flink-job jar
- val uploadResult = Request.post(s"$jmRestUrl/jars/upload")
+ val uploadResult = Request
+ .post(s"$jmRestUrl/jars/upload")
.connectTimeout(Timeout.ofSeconds(KubernetesRetriever.FLINK_REST_AWAIT_TIMEOUT_SEC))
.responseTimeout(Timeout.ofSeconds(60))
- .body(MultipartEntityBuilder
- .create()
- .addBinaryBody("jarfile", flinkJobJar, ContentType.create("application/java-archive"), flinkJobJar.getName)
- .build()).execute
+ .body(
+ MultipartEntityBuilder
+ .create()
+ .addBinaryBody(
+ "jarfile",
+ flinkJobJar,
+ ContentType.create("application/java-archive"),
+ flinkJobJar.getName)
+ .build())
+ .execute
.returnContent()
.asString(StandardCharsets.UTF_8)
@@ -72,15 +83,19 @@ object FlinkSessionSubmitHelper extends Logger {
}
if (!jarUploadResponse.isSuccessful) {
- throw new Exception(s"[flink-submit] upload flink jar to flink session cluster failed, jmRestUrl=$jmRestUrl, response=$jarUploadResponse")
+ throw new Exception(
+ s"[flink-submit] upload flink jar to flink session cluster failed, jmRestUrl=$jmRestUrl, response=$jarUploadResponse")
}
// refer to https://ci.apache.org/projects/flink/flink-docs-stable/docs/ops/rest_api/#jars-upload
- val resp = Request.post(s"$jmRestUrl/jars/${jarUploadResponse.jarId}/run")
+ val resp = Request
+ .post(s"$jmRestUrl/jars/${jarUploadResponse.jarId}/run")
.connectTimeout(Timeout.ofSeconds(KubernetesRetriever.FLINK_REST_AWAIT_TIMEOUT_SEC))
.responseTimeout(Timeout.ofSeconds(60))
.body(new StringEntity(Serialization.write(new JarRunRequest(flinkConfig))))
- .execute.returnContent().asString(StandardCharsets.UTF_8)
+ .execute
+ .returnContent()
+ .asString(StandardCharsets.UTF_8)
Try(parse(resp)) match {
case Success(ok) => (ok \ "jobid").extractOpt[String].orNull
@@ -103,14 +118,22 @@ private[client] case class JarUploadResponse(filename: String, status: String) {
/**
* refer to https://ci.apache.org/projects/flink/flink-docs-stable/docs/ops/rest_api/#jars-upload
*/
-private[client] case class JarRunRequest(entryClass: String, programArgs: String, parallelism: String, savepointPath: String, allowNonRestoredState: Boolean) {
+private[client] case class JarRunRequest(
+ entryClass: String,
+ programArgs: String,
+ parallelism: String,
+ savepointPath: String,
+ allowNonRestoredState: Boolean) {
def this(flinkConf: Configuration) {
this(
entryClass = flinkConf.get(ApplicationConfiguration.APPLICATION_MAIN_CLASS),
- programArgs = Option(flinkConf.get(ApplicationConfiguration.APPLICATION_ARGS)).map(String.join(" ", _)).orNull,
+ programArgs = Option(flinkConf.get(ApplicationConfiguration.APPLICATION_ARGS))
+ .map(String.join(" ", _))
+ .orNull,
parallelism = String.valueOf(flinkConf.get(CoreOptions.DEFAULT_PARALLELISM)),
savepointPath = flinkConf.get(SavepointConfigOptions.SAVEPOINT_PATH),
- allowNonRestoredState = flinkConf.getBoolean(SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE))
+ allowNonRestoredState =
+ flinkConf.getBoolean(SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE))
}
}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/FlinkClientTrait.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/FlinkClientTrait.scala
index ef4459bf1..5b303e588 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/FlinkClientTrait.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/FlinkClientTrait.scala
@@ -17,15 +17,14 @@
package org.apache.streampark.flink.client.`trait`
-import java.io.File
-import java.util.{Collections, List => JavaList, Map => JavaMap}
-import java.util.concurrent.TimeUnit
-import scala.collection.JavaConversions._
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
-import scala.language.postfixOps
-import scala.util.{Failure, Success, Try}
+import org.apache.streampark.common.conf.ConfigConst._
+import org.apache.streampark.common.conf.Workspace
+import org.apache.streampark.common.enums.{ApplicationType, DevelopmentMode, ExecutionMode}
+import org.apache.streampark.common.util.{DeflaterUtils, Logger}
+import org.apache.streampark.flink.client.bean._
+import org.apache.streampark.flink.core.FlinkClusterClient
+import org.apache.streampark.flink.core.conf.FlinkRunOption
+
import com.google.common.collect.Lists
import org.apache.commons.cli.{CommandLine, Options}
import org.apache.commons.collections.MapUtils
@@ -39,16 +38,19 @@ import org.apache.flink.configuration._
import org.apache.flink.runtime.jobgraph.{JobGraph, SavepointConfigOptions}
import org.apache.flink.util.FlinkException
import org.apache.flink.util.Preconditions.checkNotNull
-import org.apache.streampark.common.conf.ConfigConst._
-import org.apache.streampark.common.conf.Workspace
-import org.apache.streampark.common.enums.{ApplicationType, DevelopmentMode, ExecutionMode}
-import org.apache.streampark.common.util.{DeflaterUtils, Logger}
-import org.apache.streampark.flink.core.FlinkClusterClient
-import org.apache.streampark.flink.core.conf.FlinkRunOption
-import org.apache.streampark.flink.client.bean._
+import java.io.File
import java.time.Duration
+import java.util.{Collections, List => JavaList, Map => JavaMap}
+import java.util.concurrent.TimeUnit
+
import scala.annotation.tailrec
+import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+import scala.collection.mutable.ArrayBuffer
+import scala.language.postfixOps
+import scala.util.{Failure, Success, Try}
trait FlinkClientTrait extends Logger {
@@ -84,7 +86,9 @@ trait FlinkClientTrait extends Logger {
if (submitRequest.userJarFile != null) {
val uri = PackagedProgramUtils.resolveURI(submitRequest.userJarFile.getAbsolutePath)
val programOptions = ProgramOptions.create(commandLine)
- val executionParameters = ExecutionConfigAccessor.fromProgramOptions(programOptions, Collections.singletonList(uri.toString))
+ val executionParameters = ExecutionConfigAccessor.fromProgramOptions(
+ programOptions,
+ Collections.singletonList(uri.toString))
executionParameters.applyToConfiguration(flinkConfig)
}
@@ -97,9 +101,11 @@ trait FlinkClientTrait extends Logger {
.safeSet(ApplicationConfiguration.APPLICATION_ARGS, extractProgramArgs(submitRequest))
.safeSet(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, submitRequest.jobId)
-
- if (!submitRequest.properties.containsKey(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS.key())) {
- val flinkDefaultConfiguration = getFlinkDefaultConfiguration(submitRequest.flinkVersion.flinkHome)
+ if (
+ !submitRequest.properties.containsKey(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS.key())
+ ) {
+ val flinkDefaultConfiguration = getFlinkDefaultConfiguration(
+ submitRequest.flinkVersion.flinkHome)
// state.checkpoints.num-retained
val retainedOption = CheckpointingOptions.MAX_RETAINED_CHECKPOINTS
flinkConfig.safeSet(retainedOption, flinkDefaultConfiguration.get(retainedOption))
@@ -108,7 +114,9 @@ trait FlinkClientTrait extends Logger {
// set savepoint parameter
if (submitRequest.savePoint != null) {
flinkConfig.safeSet(SavepointConfigOptions.SAVEPOINT_PATH, submitRequest.savePoint)
- flinkConfig.setBoolean(SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE, submitRequest.allowNonRestoredState)
+ flinkConfig.setBoolean(
+ SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE,
+ submitRequest.allowNonRestoredState)
}
// set JVMOptions..
@@ -120,23 +128,26 @@ trait FlinkClientTrait extends Logger {
}
- private[this] def setJvmOptions(submitRequest: SubmitRequest, flinkConfig: Configuration): Unit = {
+ private[this] def setJvmOptions(
+ submitRequest: SubmitRequest,
+ flinkConfig: Configuration): Unit = {
if (MapUtils.isNotEmpty(submitRequest.properties)) {
- submitRequest.properties.foreach(x => {
- val k = x._1.trim
- val v = x._2.toString
- if (k == CoreOptions.FLINK_JVM_OPTIONS.key()) {
- flinkConfig.set(CoreOptions.FLINK_JVM_OPTIONS, v)
- } else if (k == CoreOptions.FLINK_JM_JVM_OPTIONS.key()) {
- flinkConfig.set(CoreOptions.FLINK_JM_JVM_OPTIONS, v)
- } else if (k == CoreOptions.FLINK_HS_JVM_OPTIONS.key()) {
- flinkConfig.set(CoreOptions.FLINK_HS_JVM_OPTIONS, v)
- } else if (k == CoreOptions.FLINK_TM_JVM_OPTIONS.key()) {
- flinkConfig.set(CoreOptions.FLINK_TM_JVM_OPTIONS, v)
- } else if (k == CoreOptions.FLINK_CLI_JVM_OPTIONS.key()) {
- flinkConfig.set(CoreOptions.FLINK_CLI_JVM_OPTIONS, v)
- }
- })
+ submitRequest.properties.foreach(
+ x => {
+ val k = x._1.trim
+ val v = x._2.toString
+ if (k == CoreOptions.FLINK_JVM_OPTIONS.key()) {
+ flinkConfig.set(CoreOptions.FLINK_JVM_OPTIONS, v)
+ } else if (k == CoreOptions.FLINK_JM_JVM_OPTIONS.key()) {
+ flinkConfig.set(CoreOptions.FLINK_JM_JVM_OPTIONS, v)
+ } else if (k == CoreOptions.FLINK_HS_JVM_OPTIONS.key()) {
+ flinkConfig.set(CoreOptions.FLINK_HS_JVM_OPTIONS, v)
+ } else if (k == CoreOptions.FLINK_TM_JVM_OPTIONS.key()) {
+ flinkConfig.set(CoreOptions.FLINK_TM_JVM_OPTIONS, v)
+ } else if (k == CoreOptions.FLINK_CLI_JVM_OPTIONS.key()) {
+ flinkConfig.set(CoreOptions.FLINK_CLI_JVM_OPTIONS, v)
+ }
+ })
}
}
@@ -178,7 +189,9 @@ trait FlinkClientTrait extends Logger {
|-------------------------------------------------------------------------------------------
|""".stripMargin)
val flinkConf = new Configuration()
- flinkConf.safeSet(ClientOptions.CLIENT_TIMEOUT, Duration.ofSeconds(cancelRequest.savePointTimeout))
+ flinkConf.safeSet(
+ ClientOptions.CLIENT_TIMEOUT,
+ Duration.ofSeconds(cancelRequest.savePointTimeout))
doCancel(cancelRequest, flinkConf)
}
@@ -186,16 +199,16 @@ trait FlinkClientTrait extends Logger {
def doSubmit(submitRequest: SubmitRequest, flinkConf: Configuration): SubmitResponse
@throws[Exception]
- def doTriggerSavepoint(request: TriggerSavepointRequest, flinkConf: Configuration): SavepointResponse
+ def doTriggerSavepoint(
+ request: TriggerSavepointRequest,
+ flinkConf: Configuration): SavepointResponse
@throws[Exception]
def doCancel(cancelRequest: CancelRequest, flinkConf: Configuration): CancelResponse
- def trySubmit(
- submitRequest: SubmitRequest,
- flinkConfig: Configuration,
- jarFile: File)(restApiFunc: (SubmitRequest, Configuration, File) => SubmitResponse)(jobGraphFunc: (SubmitRequest, Configuration, File) => SubmitResponse)
- : SubmitResponse = {
+ def trySubmit(submitRequest: SubmitRequest, flinkConfig: Configuration, jarFile: File)(
+ restApiFunc: (SubmitRequest, Configuration, File) => SubmitResponse)(
+ jobGraphFunc: (SubmitRequest, Configuration, File) => SubmitResponse): SubmitResponse = {
// Prioritize using Rest API submit while using JobGraph submit plan as backup
Try {
logInfo(s"[flink-submit] Attempting to submit in Rest API Submit Plan.")
@@ -212,16 +225,20 @@ trait FlinkClientTrait extends Logger {
}
}
- private[client] def getJobGraph(flinkConfig: Configuration, submitRequest: SubmitRequest, jarFile: File): (PackagedProgram, JobGraph) = {
- val packageProgram = PackagedProgram
- .newBuilder
+ private[client] def getJobGraph(
+ flinkConfig: Configuration,
+ submitRequest: SubmitRequest,
+ jarFile: File): (PackagedProgram, JobGraph) = {
+ val packageProgram = PackagedProgram.newBuilder
.setJarFile(jarFile)
- .setEntryPointClassName(flinkConfig.getOptional(ApplicationConfiguration.APPLICATION_MAIN_CLASS).get())
+ .setEntryPointClassName(
+ flinkConfig.getOptional(ApplicationConfiguration.APPLICATION_MAIN_CLASS).get())
.setSavepointRestoreSettings(submitRequest.savepointRestoreSettings)
.setArguments(
flinkConfig
.getOptional(ApplicationConfiguration.APPLICATION_ARGS)
- .orElse(Lists.newArrayList()): _*).build()
+ .orElse(Lists.newArrayList()): _*)
+ .build()
val jobGraph = PackagedProgramUtils.createJobGraph(
packageProgram,
@@ -239,7 +256,9 @@ trait FlinkClientTrait extends Logger {
// ----------Public Method end ------------------
- private[client] def validateAndGetActiveCommandLine(customCommandLines: JavaList[CustomCommandLine], commandLine: CommandLine): CustomCommandLine = {
+ private[client] def validateAndGetActiveCommandLine(
+ customCommandLines: JavaList[CustomCommandLine],
+ commandLine: CommandLine): CustomCommandLine = {
val line = checkNotNull(commandLine)
logInfo(s"Custom commandline: $customCommandLines")
for (cli <- customCommandLines) {
@@ -254,7 +273,9 @@ trait FlinkClientTrait extends Logger {
Try(GlobalConfiguration.loadConfiguration(s"$flinkHome/conf")).getOrElse(new Configuration())
}
- private[client] def getOptionFromDefaultFlinkConfig[T](flinkHome: String, option: ConfigOption[T]): T = {
+ private[client] def getOptionFromDefaultFlinkConfig[T](
+ flinkHome: String,
+ option: ConfigOption[T]): T = {
getFlinkDefaultConfiguration(flinkHome).get(option)
}
@@ -263,7 +284,8 @@ trait FlinkClientTrait extends Logger {
// 1. find the configuration directory
val configurationDirectory = s"$flinkHome/conf"
// 2. load the custom command lines
- val customCommandLines = loadCustomCommandLines(flinkDefaultConfiguration, configurationDirectory)
+ val customCommandLines =
+ loadCustomCommandLines(flinkDefaultConfiguration, configurationDirectory)
new CliFrontend(flinkDefaultConfiguration, customCommandLines)
customCommandLines
}
@@ -272,31 +294,36 @@ trait FlinkClientTrait extends Logger {
if (submitRequest.properties.containsKey(KEY_FLINK_PARALLELISM())) {
Integer.valueOf(submitRequest.properties.get(KEY_FLINK_PARALLELISM()).toString)
} else {
- getFlinkDefaultConfiguration(submitRequest.flinkVersion.flinkHome).getInteger(
- CoreOptions.DEFAULT_PARALLELISM,
- CoreOptions.DEFAULT_PARALLELISM.defaultValue())
+ getFlinkDefaultConfiguration(submitRequest.flinkVersion.flinkHome)
+ .getInteger(CoreOptions.DEFAULT_PARALLELISM, CoreOptions.DEFAULT_PARALLELISM.defaultValue())
}
}
- private[this] def getCommandLineAndFlinkConfig(submitRequest: SubmitRequest): (CommandLine, Configuration) = {
+ private[this] def getCommandLineAndFlinkConfig(
+ submitRequest: SubmitRequest): (CommandLine, Configuration) = {
val commandLineOptions = getCommandLineOptions(submitRequest.flinkVersion.flinkHome)
// read and verify user config...
val cliArgs = {
val optionMap = new mutable.HashMap[String, Any]()
- submitRequest.appOption.filter(x => {
- // 验证参数是否合法...
- val verify = commandLineOptions.hasOption(x._1)
- if (!verify) logWarn(s"param:${x._1} is error,skip it.")
- verify
- }).foreach(x => {
- val opt = commandLineOptions.getOption(x._1.trim).getOpt
- Try(x._2.toBoolean).getOrElse(x._2) match {
- case b if b.isInstanceOf[Boolean] => if (b.asInstanceOf[Boolean]) optionMap += s"-$opt" -> true
- case v => optionMap += s"-$opt" -> v
- }
- })
+ submitRequest.appOption
+ .filter(
+ x => {
+ // 验证参数是否合法...
+ val verify = commandLineOptions.hasOption(x._1)
+ if (!verify) logWarn(s"param:${x._1} is error,skip it.")
+ verify
+ })
+ .foreach(
+ x => {
+ val opt = commandLineOptions.getOption(x._1.trim).getOpt
+ Try(x._2.toBoolean).getOrElse(x._2) match {
+ case b if b.isInstanceOf[Boolean] =>
+ if (b.asInstanceOf[Boolean]) optionMap += s"-$opt" -> true
+ case v => optionMap += s"-$opt" -> v
+ }
+ })
// fromSavePoint
if (submitRequest.savePoint != null) {
@@ -309,21 +336,23 @@ trait FlinkClientTrait extends Logger {
}
val array = new ArrayBuffer[String]()
- optionMap.foreach(x => {
- array += x._1
- x._2 match {
- case v: String => array += v
- case _ =>
- }
- })
+ optionMap.foreach(
+ x => {
+ array += x._1
+ x._2 match {
+ case v: String => array += v
+ case _ =>
+ }
+ })
// app properties
if (MapUtils.isNotEmpty(submitRequest.properties)) {
- submitRequest.properties.foreach(x => {
- if (!x._1.startsWith(CoreOptions.FLINK_JVM_OPTIONS.key())) {
- array += s"-D${x._1}=${x._2}"
- }
- })
+ submitRequest.properties.foreach(
+ x => {
+ if (!x._1.startsWith(CoreOptions.FLINK_JVM_OPTIONS.key())) {
+ array += s"-D${x._1}=${x._2}"
+ }
+ })
}
array.toArray
}
@@ -334,9 +363,12 @@ trait FlinkClientTrait extends Logger {
val commandLine = FlinkRunOption.parse(commandLineOptions, cliArgs, true)
- val activeCommandLine = validateAndGetActiveCommandLine(getCustomCommandLines(submitRequest.flinkVersion.flinkHome), commandLine)
+ val activeCommandLine = validateAndGetActiveCommandLine(
+ getCustomCommandLines(submitRequest.flinkVersion.flinkHome),
+ commandLine)
- val configuration = applyConfiguration(submitRequest.flinkVersion.flinkHome, activeCommandLine, commandLine)
+ val configuration =
+ applyConfiguration(submitRequest.flinkVersion.flinkHome, activeCommandLine, commandLine)
commandLine -> configuration
@@ -352,7 +384,9 @@ trait FlinkClientTrait extends Logger {
FlinkRunOption.mergeOptions(CliFrontendParser.getRunCommandOptions, customCommandLineOptions)
}
- private[client] def extractConfiguration(flinkHome: String, properties: JavaMap[String, Any]): Configuration = {
+ private[client] def extractConfiguration(
+ flinkHome: String,
+ properties: JavaMap[String, Any]): Configuration = {
val commandLine = {
val commandLineOptions = getCommandLineOptions(flinkHome)
// read and verify user config...
@@ -366,7 +400,8 @@ trait FlinkClientTrait extends Logger {
}
FlinkRunOption.parse(commandLineOptions, cliArgs, true)
}
- val activeCommandLine = validateAndGetActiveCommandLine(getCustomCommandLines(flinkHome), commandLine)
+ val activeCommandLine =
+ validateAndGetActiveCommandLine(getCustomCommandLines(flinkHome), commandLine)
val flinkConfig = applyConfiguration(flinkHome, activeCommandLine, commandLine)
flinkConfig
}
@@ -448,19 +483,23 @@ trait FlinkClientTrait extends Logger {
programArgs.toList.asJava
}
- private[this] def applyConfiguration(flinkHome: String, activeCustomCommandLine: CustomCommandLine, commandLine: CommandLine): Configuration = {
+ private[this] def applyConfiguration(
+ flinkHome: String,
+ activeCustomCommandLine: CustomCommandLine,
+ commandLine: CommandLine): Configuration = {
require(activeCustomCommandLine != null, "activeCustomCommandLine must not be null.")
val executorConfig = activeCustomCommandLine.toConfiguration(commandLine)
val customConfiguration = new Configuration(executorConfig)
val configuration = new Configuration()
val flinkDefaultConfiguration = getFlinkDefaultConfiguration(flinkHome)
- flinkDefaultConfiguration.keySet.foreach(x => {
- flinkDefaultConfiguration.getString(x, null) match {
- case v if v != null => configuration.setString(x, v)
- case _ =>
- }
- })
+ flinkDefaultConfiguration.keySet.foreach(
+ x => {
+ flinkDefaultConfiguration.getString(x, null) match {
+ case v if v != null => configuration.setString(x, v)
+ case _ =>
+ }
+ })
configuration.addAll(customConfiguration)
configuration
}
@@ -474,45 +513,63 @@ trait FlinkClientTrait extends Logger {
}
}
- private[client] def cancelJob(cancelRequest: CancelRequest, jobID: JobID, client: ClusterClient[_]): String = {
+ private[client] def cancelJob(
+ cancelRequest: CancelRequest,
+ jobID: JobID,
+ client: ClusterClient[_]): String = {
val savePointDir: String = tryGetSavepointPathIfNeed(cancelRequest)
val clientWrapper = new FlinkClusterClient(client)
- (Try(cancelRequest.withSavepoint).getOrElse(false), Try(cancelRequest.withDrain).getOrElse(false)) match {
+ (
+ Try(cancelRequest.withSavepoint).getOrElse(false),
+ Try(cancelRequest.withDrain).getOrElse(false)) match {
case (false, false) =>
client.cancel(jobID).get()
null
- case (true, false) => clientWrapper.cancelWithSavepoint(jobID, savePointDir).get(cancelRequest.savePointTimeout, TimeUnit.SECONDS)
- case (_, _) => clientWrapper.stopWithSavepoint(jobID, cancelRequest.withDrain, savePointDir).get(cancelRequest.savePointTimeout, TimeUnit.SECONDS)
+ case (true, false) =>
+ clientWrapper
+ .cancelWithSavepoint(jobID, savePointDir)
+ .get(cancelRequest.savePointTimeout, TimeUnit.SECONDS)
+ case (_, _) =>
+ clientWrapper
+ .stopWithSavepoint(jobID, cancelRequest.withDrain, savePointDir)
+ .get(cancelRequest.savePointTimeout, TimeUnit.SECONDS)
}
}
private def tryGetSavepointPathIfNeed(request: SavepointRequestTrait): String = {
- if (!request.withSavepoint) null else {
+ if (!request.withSavepoint) null
+ else {
if (StringUtils.isNotEmpty(request.savepointPath)) {
request.savepointPath
} else {
val configDir = getOptionFromDefaultFlinkConfig[String](
request.flinkVersion.flinkHome,
- ConfigOptions.key(CheckpointingOptions.SAVEPOINT_DIRECTORY.key())
+ ConfigOptions
+ .key(CheckpointingOptions.SAVEPOINT_DIRECTORY.key())
.stringType()
.defaultValue {
if (request.executionMode == ExecutionMode.YARN_APPLICATION) {
Workspace.remote.APP_SAVEPOINTS
} else null
- })
+ }
+ )
if (StringUtils.isEmpty(configDir)) {
- throw new FlinkException(s"[StreamPark] executionMode: ${request.executionMode.getName}, savePoint path is null or invalid.")
+ throw new FlinkException(
+ s"[StreamPark] executionMode: ${request.executionMode.getName}, savePoint path is null or invalid.")
} else configDir
}
}
}
- private[client] def triggerSavepoint(savepointRequest: TriggerSavepointRequest, jobID: JobID, client: ClusterClient[_]): String = {
+ private[client] def triggerSavepoint(
+ savepointRequest: TriggerSavepointRequest,
+ jobID: JobID,
+ client: ClusterClient[_]): String = {
val savepointPath = tryGetSavepointPathIfNeed(savepointRequest)
val clientWrapper = new FlinkClusterClient(client)
clientWrapper.triggerSavepoint(jobID, savepointPath).get()
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.scala
index 65b627ded..27f5fe7a5 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.scala
@@ -17,8 +17,11 @@
package org.apache.streampark.flink.client.`trait`
-import javax.annotation.Nonnull
-import scala.language.postfixOps
+import org.apache.streampark.common.enums.{ExecutionMode, FlinkK8sRestExposedType}
+import org.apache.streampark.flink.client.bean._
+import org.apache.streampark.flink.kubernetes.IngressController
+import org.apache.streampark.flink.packer.pipeline.DockerImageBuildResponse
+
import org.apache.commons.lang3.StringUtils
import org.apache.flink.api.common.JobID
import org.apache.flink.client.deployment.ClusterSpecification
@@ -27,14 +30,12 @@ import org.apache.flink.configuration._
import org.apache.flink.kubernetes.{KubernetesClusterClientFactory, KubernetesClusterDescriptor}
import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions
import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions.ServiceExposedType
-import org.apache.streampark.common.enums.{ExecutionMode, FlinkK8sRestExposedType}
-import org.apache.streampark.flink.kubernetes.IngressController
-import org.apache.streampark.flink.client.bean._
-import org.apache.streampark.flink.packer.pipeline.DockerImageBuildResponse
-/**
- * kubernetes native mode submit
- */
+import javax.annotation.Nonnull
+
+import scala.language.postfixOps
+
+/** kubernetes native mode submit */
trait KubernetesNativeClientTrait extends FlinkClientTrait {
override def setConfig(submitRequest: SubmitRequest, flinkConfig: Configuration): Unit = {
@@ -49,48 +50,56 @@ trait KubernetesNativeClientTrait extends FlinkClientTrait {
if (submitRequest.buildResult != null) {
if (submitRequest.executionMode == ExecutionMode.KUBERNETES_NATIVE_APPLICATION) {
val buildResult = submitRequest.buildResult.asInstanceOf[DockerImageBuildResponse]
- buildResult.podTemplatePaths.foreach(p => {
- flinkConfig
- .safeSet(KubernetesConfigOptions.KUBERNETES_POD_TEMPLATE, p._2)
- .safeSet(KubernetesConfigOptions.JOB_MANAGER_POD_TEMPLATE, p._2)
- .safeSet(KubernetesConfigOptions.TASK_MANAGER_POD_TEMPLATE, p._2)
- })
+ buildResult.podTemplatePaths.foreach(
+ p => {
+ flinkConfig
+ .safeSet(KubernetesConfigOptions.KUBERNETES_POD_TEMPLATE, p._2)
+ .safeSet(KubernetesConfigOptions.JOB_MANAGER_POD_TEMPLATE, p._2)
+ .safeSet(KubernetesConfigOptions.TASK_MANAGER_POD_TEMPLATE, p._2)
+ })
}
}
// add flink conf configuration, mainly to set the log4j configuration
if (!flinkConfig.contains(DeploymentOptionsInternal.CONF_DIR)) {
- flinkConfig.safeSet(DeploymentOptionsInternal.CONF_DIR, s"${submitRequest.flinkVersion.flinkHome}/conf")
+ flinkConfig.safeSet(
+ DeploymentOptionsInternal.CONF_DIR,
+ s"${submitRequest.flinkVersion.flinkHome}/conf")
}
if (flinkConfig.get(KubernetesConfigOptions.NAMESPACE).isEmpty) {
flinkConfig.removeConfig(KubernetesConfigOptions.NAMESPACE)
}
- logInfo(
- s"""
- |------------------------------------------------------------------
- |Effective submit configuration: $flinkConfig
- |------------------------------------------------------------------
- |""".stripMargin)
+ logInfo(s"""
+ |------------------------------------------------------------------
+ |Effective submit configuration: $flinkConfig
+ |------------------------------------------------------------------
+ |""".stripMargin)
}
// Tip: Perhaps it would be better to let users freely specify the savepoint directory
@throws[Exception]
- override def doCancel(cancelRequest: CancelRequest, flinkConfig: Configuration): CancelResponse = {
- executeClientAction(cancelRequest, flinkConfig, (jobId, clusterClient) => {
- val actionResult = super.cancelJob(cancelRequest, jobId, clusterClient)
- CancelResponse(actionResult)
- })
+ override def doCancel(
+ cancelRequest: CancelRequest,
+ flinkConfig: Configuration): CancelResponse = {
+ executeClientAction(
+ cancelRequest,
+ flinkConfig,
+ (jobId, clusterClient) => {
+ val actionResult = super.cancelJob(cancelRequest, jobId, clusterClient)
+ CancelResponse(actionResult)
+ })
}
- private[this] def executeClientAction[O, R <: SavepointRequestTrait](request: R,
- flinkConfig: Configuration,
- actFunc: (JobID, ClusterClient[_]) => O): O = {
+ private[this] def executeClientAction[O, R <: SavepointRequestTrait](
+ request: R,
+ flinkConfig: Configuration,
+ actFunc: (JobID, ClusterClient[_]) => O): O = {
val hints = s"[flink-client] execute ${request.getClass.getSimpleName} for flink job failed,"
require(
StringUtils.isNotBlank(request.clusterId),
- s"${hints}, clusterId is null, mode=${flinkConfig.get(DeploymentOptions.TARGET)}")
+ s"$hints, clusterId is null, mode=${flinkConfig.get(DeploymentOptions.TARGET)}")
flinkConfig
.safeSet(KubernetesConfigOptions.CLUSTER_ID, request.clusterId)
@@ -101,11 +110,13 @@ trait KubernetesNativeClientTrait extends FlinkClientTrait {
try {
clusterDescriptor = getK8sClusterDescriptor(flinkConfig)
- client = clusterDescriptor.retrieve(flinkConfig.getString(KubernetesConfigOptions.CLUSTER_ID)).getClusterClient
+ client = clusterDescriptor
+ .retrieve(flinkConfig.getString(KubernetesConfigOptions.CLUSTER_ID))
+ .getClusterClient
actFunc(JobID.fromHexString(request.jobId), client)
} catch {
case e: Exception =>
- logger.error(s"$hints mode=${flinkConfig.get(DeploymentOptions.TARGET)}, request=${request}")
+ logger.error(s"$hints mode=${flinkConfig.get(DeploymentOptions.TARGET)}, request=$request")
throw e
} finally {
if (client != null) client.close()
@@ -114,11 +125,16 @@ trait KubernetesNativeClientTrait extends FlinkClientTrait {
}
@throws[Exception]
- override def doTriggerSavepoint(request: TriggerSavepointRequest, flinkConfig: Configuration): SavepointResponse = {
- executeClientAction(request, flinkConfig, (jobId, clusterClient) => {
- val actionResult = super.triggerSavepoint(request, jobId, clusterClient)
- SavepointResponse(actionResult)
- })
+ override def doTriggerSavepoint(
+ request: TriggerSavepointRequest,
+ flinkConfig: Configuration): SavepointResponse = {
+ executeClientAction(
+ request,
+ flinkConfig,
+ (jobId, clusterClient) => {
+ val actionResult = super.triggerSavepoint(request, jobId, clusterClient)
+ SavepointResponse(actionResult)
+ })
}
// noinspection DuplicatedCode
@@ -132,7 +148,8 @@ trait KubernetesNativeClientTrait extends FlinkClientTrait {
the kubernetes context contents in the FlinkKubeClient.
2. Specify an explicit key kubernetes.config.file in flinkConfig instead of the default value.
*/
- def getK8sClusterDescriptorAndSpecification(flinkConfig: Configuration): (KubernetesClusterDescriptor, ClusterSpecification) = {
+ def getK8sClusterDescriptorAndSpecification(
+ flinkConfig: Configuration): (KubernetesClusterDescriptor, ClusterSpecification) = {
val clientFactory = new KubernetesClusterClientFactory()
val clusterDescriptor = clientFactory.createClusterDescriptor(flinkConfig)
val clusterSpecification = clientFactory.getClusterSpecification(flinkConfig)
@@ -146,15 +163,17 @@ trait KubernetesNativeClientTrait extends FlinkClientTrait {
}
protected def flinkConfIdentifierInfo(@Nonnull conf: Configuration): String =
- s"executionMode=${conf.get(DeploymentOptions.TARGET)}, clusterId=${conf.get(KubernetesConfigOptions.CLUSTER_ID)}, " +
+ s"executionMode=${conf.get(DeploymentOptions.TARGET)}, clusterId=${conf.get(
+ KubernetesConfigOptions.CLUSTER_ID)}, " +
s"namespace=${conf.get(KubernetesConfigOptions.NAMESPACE)}"
- private def covertToServiceExposedType(exposedType: FlinkK8sRestExposedType): ServiceExposedType = exposedType match {
- case FlinkK8sRestExposedType.CLUSTER_IP => ServiceExposedType.ClusterIP
- case FlinkK8sRestExposedType.LOAD_BALANCER => ServiceExposedType.LoadBalancer
- case FlinkK8sRestExposedType.NODE_PORT => ServiceExposedType.NodePort
- case _ => ServiceExposedType.LoadBalancer
- }
+ private def covertToServiceExposedType(exposedType: FlinkK8sRestExposedType): ServiceExposedType =
+ exposedType match {
+ case FlinkK8sRestExposedType.CLUSTER_IP => ServiceExposedType.ClusterIP
+ case FlinkK8sRestExposedType.LOAD_BALANCER => ServiceExposedType.LoadBalancer
+ case FlinkK8sRestExposedType.NODE_PORT => ServiceExposedType.NodePort
+ case _ => ServiceExposedType.LoadBalancer
+ }
def getDefaultKubernetesConf(k8sConf: String): String = {
val homePath: String = System.getProperty("user.home")
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/YarnClientTrait.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/YarnClientTrait.scala
index ec76c34aa..fa88df7dd 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/YarnClientTrait.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/scala/org/apache/streampark/flink/client/trait/YarnClientTrait.scala
@@ -17,11 +17,10 @@
package org.apache.streampark.flink.client.`trait`
-import org.apache.flink.api.common.JobID
+import org.apache.streampark.common.util.Utils
+import org.apache.streampark.flink.client.bean._
-import java.lang.{Boolean => JavaBool}
-import java.lang.reflect.Method
-import scala.util.Try
+import org.apache.flink.api.common.JobID
import org.apache.flink.client.deployment.{ClusterDescriptor, ClusterSpecification, DefaultClusterClientServiceLoader}
import org.apache.flink.client.program.{ClusterClient, ClusterClientProvider}
import org.apache.flink.configuration.Configuration
@@ -30,16 +29,19 @@ import org.apache.flink.util.FlinkException
import org.apache.flink.yarn.{YarnClusterClientFactory, YarnClusterDescriptor}
import org.apache.flink.yarn.configuration.YarnConfigOptions
import org.apache.hadoop.yarn.api.records.ApplicationId
-import org.apache.streampark.common.util.Utils
-import org.apache.streampark.flink.client.bean._
-/**
- * yarn application mode submit
- */
+import java.lang.{Boolean => JavaBool}
+import java.lang.reflect.Method
+
+import scala.util.Try
+
+/** yarn application mode submit */
trait YarnClientTrait extends FlinkClientTrait {
- private[this] def executeClientAction[R <: SavepointRequestTrait, O](request: R, flinkConf: Configuration,
- actionFunc: (JobID, ClusterClient[_]) => O): O = {
+ private[this] def executeClientAction[R <: SavepointRequestTrait, O](
+ request: R,
+ flinkConf: Configuration,
+ actionFunc: (JobID, ClusterClient[_]) => O): O = {
val jobID = getJobID(request.jobId)
val clusterClient = {
flinkConf.safeSet(YarnConfigOptions.APPLICATION_ID, request.clusterId)
@@ -55,22 +57,31 @@ trait YarnClientTrait extends FlinkClientTrait {
Try {
actionFunc(jobID, clusterClient)
}.recover {
- case e => throw new FlinkException(
- s"[StreamPark] Do ${request.getClass.getSimpleName} for the job ${request.jobId} failed. " +
- s"detail: ${Utils.stringifyException(e)}");
+ case e =>
+ throw new FlinkException(
+ s"[StreamPark] Do ${request.getClass.getSimpleName} for the job ${request.jobId} failed. " +
+ s"detail: ${Utils.stringifyException(e)}");
}.get
}
- override def doTriggerSavepoint(request: TriggerSavepointRequest, flinkConf: Configuration): SavepointResponse = {
- executeClientAction(request, flinkConf, (jid, client) => {
- SavepointResponse(super.triggerSavepoint(request, jid, client))
- })
+ override def doTriggerSavepoint(
+ request: TriggerSavepointRequest,
+ flinkConf: Configuration): SavepointResponse = {
+ executeClientAction(
+ request,
+ flinkConf,
+ (jid, client) => {
+ SavepointResponse(super.triggerSavepoint(request, jid, client))
+ })
}
override def doCancel(cancelRequest: CancelRequest, flinkConf: Configuration): CancelResponse = {
- executeClientAction(cancelRequest, flinkConf, (jid, client) => {
- CancelResponse(super.cancelJob(cancelRequest, jid, client))
- })
+ executeClientAction(
+ cancelRequest,
+ flinkConf,
+ (jid, client) => {
+ CancelResponse(super.cancelJob(cancelRequest, jid, client))
+ })
}
private lazy val deployInternalMethod: Method = {
@@ -81,7 +92,8 @@ trait YarnClientTrait extends FlinkClientTrait {
classOf[JobGraph],
Boolean2boolean(true).getClass // get boolean class.
)
- val deployInternal = classOf[YarnClusterDescriptor].getDeclaredMethod("deployInternal", paramClass: _*)
+ val deployInternal =
+ classOf[YarnClusterDescriptor].getDeclaredMethod("deployInternal", paramClass: _*)
deployInternal.setAccessible(true)
deployInternal
}
@@ -93,16 +105,19 @@ trait YarnClientTrait extends FlinkClientTrait {
yarnClusterEntrypoint: String,
jobGraph: JobGraph,
detached: JavaBool): ClusterClientProvider[ApplicationId] = {
- deployInternalMethod.invoke(
- clusterDescriptor,
- clusterSpecification,
- applicationName,
- yarnClusterEntrypoint,
- jobGraph,
- detached).asInstanceOf[ClusterClientProvider[ApplicationId]]
+ deployInternalMethod
+ .invoke(
+ clusterDescriptor,
+ clusterSpecification,
+ applicationName,
+ yarnClusterEntrypoint,
+ jobGraph,
+ detached)
+ .asInstanceOf[ClusterClientProvider[ApplicationId]]
}
- private[client] def getSessionClusterDescriptor[T <: ClusterDescriptor[ApplicationId]](flinkConfig: Configuration): (ApplicationId, T) = {
+ private[client] def getSessionClusterDescriptor[T <: ClusterDescriptor[ApplicationId]](
+ flinkConfig: Configuration): (ApplicationId, T) = {
val serviceLoader = new DefaultClusterClientServiceLoader
val clientFactory = serviceLoader.getClusterClientFactory[ApplicationId](flinkConfig)
val yarnClusterId: ApplicationId = clientFactory.getClusterId(flinkConfig)
@@ -111,7 +126,8 @@ trait YarnClientTrait extends FlinkClientTrait {
(yarnClusterId, clusterDescriptor)
}
- private[client] def getSessionClusterDeployDescriptor[T <: ClusterDescriptor[ApplicationId]](flinkConfig: Configuration): (ClusterSpecification, T) = {
+ private[client] def getSessionClusterDeployDescriptor[T <: ClusterDescriptor[ApplicationId]](
+ flinkConfig: Configuration): (ClusterSpecification, T) = {
val serviceLoader = new DefaultClusterClientServiceLoader
val clientFactory = serviceLoader.getClusterClientFactory[ApplicationId](flinkConfig)
val clusterSpecification = clientFactory.getClusterSpecification(flinkConfig)
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/scala/org/apache/streampark/flink/client/test/ParameterTestCase.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/scala/org/apache/streampark/flink/client/test/ParameterTestCase.scala
index 46c773f0c..32929d2d2 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/scala/org/apache/streampark/flink/client/test/ParameterTestCase.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/scala/org/apache/streampark/flink/client/test/ParameterTestCase.scala
@@ -16,17 +16,16 @@
*/
package org.apache.streampark.flink.client.test
-import scala.collection.mutable.ArrayBuffer
import org.apache.commons.lang3.StringUtils
import org.apache.flink.api.java.utils.ParameterTool
import org.junit.jupiter.api.{Assertions, Test}
+import scala.collection.mutable.ArrayBuffer
+
class ParameterTestCase {
@Test def readArgs(): Unit = {
- val arg = Array(
- "--flink.deployment.option.parallelism",
- "10")
+ val arg = Array("--flink.deployment.option.parallelism", "10")
val args = Array(
"--flink.home",
"hdfs://nameservice1/streampark/flink/flink-1.11.1",
@@ -36,7 +35,9 @@ class ParameterTestCase {
"5")
val param = ParameterTool.fromArgs(arg).mergeWith(ParameterTool.fromArgs(args))
- Assertions.assertEquals("hdfs://nameservice1/streampark/flink/flink-1.11.1", param.get("flink.home"))
+ Assertions.assertEquals(
+ "hdfs://nameservice1/streampark/flink/flink-1.11.1",
+ param.get("flink.home"))
Assertions.assertEquals("testApp123", param.get("app.name"))
Assertions.assertEquals("5", param.get("flink.deployment.option.parallelism"))
}
diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/scala/org/apache/streampark/flink/client/test/YarnPerJobTestCase.scala b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/scala/org/apache/streampark/flink/client/test/YarnPerJobTestCase.scala
index 5b3e03560..6451d5908 100644
--- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/scala/org/apache/streampark/flink/client/test/YarnPerJobTestCase.scala
+++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/scala/org/apache/streampark/flink/client/test/YarnPerJobTestCase.scala
@@ -16,12 +16,9 @@
*/
package org.apache.streampark.flink.client.test
-import java.io.File
-import java.lang.reflect.Method
-import java.util
-
-import scala.collection.JavaConversions._
-import scala.collection.JavaConverters._
+import org.apache.streampark.common.util.Logger
+import org.apache.streampark.flink.client.bean
+import org.apache.streampark.flink.core.conf.FlinkRunOption
import org.apache.commons.cli.Options
import org.apache.flink.client.cli.{CliFrontendParser, CustomCommandLine}
@@ -39,37 +36,30 @@ import org.apache.flink.yarn.entrypoint.YarnJobClusterEntrypoint
import org.apache.hadoop.fs.{Path => HadoopPath}
import org.apache.hadoop.yarn.api.records.ApplicationId
-import org.apache.streampark.common.util.Logger
-import org.apache.streampark.flink.core.conf.FlinkRunOption
-import org.apache.streampark.flink.client.bean
+import java.io.File
+import java.lang.reflect.Method
+import java.util
-/**
- * perJob to submit jobs programmatically,
- */
+import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
+
+/** perJob to submit jobs programmatically, */
object YarnPerJobTestCase extends Logger {
- /**
- * You must install and deploy flink locally, and configure FLINK_HOME
- */
+ /** You must install and deploy flink locally, and configure FLINK_HOME */
lazy val FLINK_HOME = {
val flinkLocalHome = System.getenv("FLINK_HOME")
logInfo(s"flinkHome: $flinkLocalHome")
flinkLocalHome
}
- /**
- * SocketWindowWordCount.jar Download the built-in sample program in flink/examples
- */
+ /** SocketWindowWordCount.jar Download the built-in sample program in flink/examples */
val userJar = s"$FLINK_HOME/examples/streaming/SocketWindowWordCount.jar"
- /**
- * Parameters required to run the program
- */
+ /** Parameters required to run the program */
val programArgs = "--hostname localhost --port 9999"
- /**
- * Run the specified option parameter
- */
+ /** Run the specified option parameter */
val option = "-e yarn-per-job -p 2 -n"
lazy val flinkDefaultConfiguration: Configuration = {
@@ -86,8 +76,8 @@ object YarnPerJobTestCase extends Logger {
}
/**
- * The private method deployInternal of YarnClusterDescriptor is reflected, mainly to pass in applicationName.
- * In the original method, applicationName is hard-coded.
+ * The private method deployInternal of YarnClusterDescriptor is reflected, mainly to pass in
+ * applicationName. In the original method, applicationName is hard-coded.
*/
lazy val deployInternalMethod: Method = {
val paramClass = Array(
@@ -96,7 +86,8 @@ object YarnPerJobTestCase extends Logger {
classOf[String],
classOf[JobGraph],
Boolean2boolean(true).getClass)
- val deployInternal = classOf[YarnClusterDescriptor].getDeclaredMethod("deployInternal", paramClass: _*)
+ val deployInternal =
+ classOf[YarnClusterDescriptor].getDeclaredMethod("deployInternal", paramClass: _*)
deployInternal.setAccessible(true)
deployInternal
}
@@ -108,13 +99,15 @@ object YarnPerJobTestCase extends Logger {
yarnClusterEntrypoint: String,
jobGraph: JobGraph,
detached: java.lang.Boolean): ClusterClientProvider[ApplicationId] = {
- deployInternalMethod.invoke(
- clusterDescriptor,
- clusterSpecification,
- applicationName,
- yarnClusterEntrypoint,
- jobGraph,
- detached).asInstanceOf[ClusterClientProvider[ApplicationId]]
+ deployInternalMethod
+ .invoke(
+ clusterDescriptor,
+ clusterSpecification,
+ applicationName,
+ yarnClusterEntrypoint,
+ jobGraph,
+ detached)
+ .asInstanceOf[ClusterClientProvider[ApplicationId]]
}
def main(args: Array[String]): Unit = {
@@ -125,7 +118,9 @@ object YarnPerJobTestCase extends Logger {
customCommandLine.addGeneralOptions(customCommandLineOptions)
customCommandLine.addRunOptions(customCommandLineOptions)
}
- val commandLineOptions = FlinkRunOption.mergeOptions(CliFrontendParser.getRunCommandOptions, customCommandLineOptions)
+ val commandLineOptions = FlinkRunOption.mergeOptions(
+ CliFrontendParser.getRunCommandOptions,
+ customCommandLineOptions)
val cliArgs = option.split("\\s+")
FlinkRunOption.parse(commandLineOptions, cliArgs, true)
}
@@ -139,20 +134,33 @@ object YarnPerJobTestCase extends Logger {
val executorConfig = checkNotNull(activeCommandLine).toConfiguration(commandLine)
val flinkConfig = new Configuration(executorConfig)
flinkConfig.set(DeploymentOptions.TARGET, YarnDeploymentTarget.PER_JOB.getName)
- flinkConfig.set(ApplicationConfiguration.APPLICATION_ARGS, programArgs.split("\\s+").toList.asJava)
- flinkConfig.set(JobManagerOptions.TOTAL_FLINK_MEMORY, MemorySize.parse("1024", MemoryUnit.MEGA_BYTES))
- flinkConfig.set(TaskManagerOptions.TOTAL_FLINK_MEMORY, MemorySize.parse("1024", MemoryUnit.MEGA_BYTES))
+ flinkConfig.set(
+ ApplicationConfiguration.APPLICATION_ARGS,
+ programArgs.split("\\s+").toList.asJava)
+ flinkConfig.set(
+ JobManagerOptions.TOTAL_FLINK_MEMORY,
+ MemorySize.parse("1024", MemoryUnit.MEGA_BYTES))
+ flinkConfig.set(
+ TaskManagerOptions.TOTAL_FLINK_MEMORY,
+ MemorySize.parse("1024", MemoryUnit.MEGA_BYTES))
val clusterClientServiceLoader = new DefaultClusterClientServiceLoader
- val clientFactory = clusterClientServiceLoader.getClusterClientFactory[ApplicationId](flinkConfig)
+ val clientFactory =
+ clusterClientServiceLoader.getClusterClientFactory[ApplicationId](flinkConfig)
val clusterDescriptor = {
- val clusterDescriptor = clientFactory.createClusterDescriptor(flinkConfig).asInstanceOf[YarnClusterDescriptor]
- val flinkDistJar = new File(s"$FLINK_HOME/lib").list().filter(_.matches("flink-dist.*\\.jar")) match {
- case Array() => throw new IllegalArgumentException(s"[StreamPark] can no found flink-dist jar in $FLINK_HOME/lib")
- case array if array.length == 1 => s"$FLINK_HOME/lib/${array.head}"
- case more => throw new IllegalArgumentException(s"[StreamPark] found multiple flink-dist jar in $FLINK_HOME/lib,[${more.mkString(",")}]")
- }
+ val clusterDescriptor =
+ clientFactory.createClusterDescriptor(flinkConfig).asInstanceOf[YarnClusterDescriptor]
+ val flinkDistJar =
+ new File(s"$FLINK_HOME/lib").list().filter(_.matches("flink-dist.*\\.jar")) match {
+ case Array() =>
+ throw new IllegalArgumentException(
+ s"[StreamPark] can no found flink-dist jar in $FLINK_HOME/lib")
+ case array if array.length == 1 => s"$FLINK_HOME/lib/${array.head}"
+ case more =>
+ throw new IllegalArgumentException(
+ s"[StreamPark] found multiple flink-dist jar in $FLINK_HOME/lib,[${more.mkString(",")}]")
+ }
clusterDescriptor.setLocalJarPath(new HadoopPath(flinkDistJar))
clusterDescriptor
}
@@ -163,16 +171,11 @@ object YarnPerJobTestCase extends Logger {
logInfo("------------------<<specification>>------------------")
logInfo(s"$clusterSpecification")
logInfo("------------------------------------")
- val packagedProgram = PackagedProgram
- .newBuilder
+ val packagedProgram = PackagedProgram.newBuilder
.setJarFile(new File(userJar))
.setArguments(programArgs.split("\\s"): _*)
.build
- val jobGraph = PackagedProgramUtils.createJobGraph(
- packagedProgram,
- flinkConfig,
- 1,
- false)
+ val jobGraph = PackagedProgramUtils.createJobGraph(packagedProgram, flinkConfig, 1, false)
logInfo("------------------<<jobId>>------------------")
logger.info(s"${jobGraph.getJobID.toString}")
logInfo("------------------------------------")
@@ -190,7 +193,8 @@ object YarnPerJobTestCase extends Logger {
logInfo(s"Flink Job Started: applicationId: $applicationId ")
logInfo("-------------------------------------")
bean.SubmitResponse(applicationId.toString, flinkConfig.toMap)
- } finally if (clusterDescriptor != null) {
+ } finally
+ if (clusterDescriptor != null) {
clusterDescriptor.close()
}
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/conf/ThresholdConf.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/conf/ThresholdConf.scala
index 559c6814b..529fc9d20 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/conf/ThresholdConf.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/conf/ThresholdConf.scala
@@ -17,14 +17,14 @@
package org.apache.streampark.flink.connector.conf
+import org.apache.streampark.common.util.ConfigUtils
+import org.apache.streampark.flink.connector.conf.FailoverStorageType.{Console, FailoverStorageType, Kafka, MySQL, NONE}
+
import java.util.Properties
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
-import org.apache.streampark.common.util.ConfigUtils
-import org.apache.streampark.flink.connector.conf.FailoverStorageType.{Console, FailoverStorageType, Kafka, MySQL, NONE}
-
case class ThresholdConf(prefixStr: String, parameters: Properties) {
private val option: ThresholdConfigOption = ThresholdConfigOption(prefixStr, parameters)
@@ -43,7 +43,9 @@ case class ThresholdConf(prefixStr: String, parameters: Properties) {
case Console | NONE => null
case Kafka => ConfigUtils.getConf(parameters.toMap.asJava, "failover.kafka.")
case MySQL => ConfigUtils.getConf(parameters.toMap.asJava, "failover.mysql.")
- case _ => throw new IllegalArgumentException(s"[StreamPark] usage error! failover.storage must not be null! ")
+ case _ =>
+ throw new IllegalArgumentException(
+ s"[StreamPark] usage error! failover.storage must not be null! ")
}
}
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/conf/ThresholdConfigOption.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/conf/ThresholdConfigOption.scala
index a8477705f..b2924b1e7 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/conf/ThresholdConfigOption.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/conf/ThresholdConfigOption.scala
@@ -17,14 +17,15 @@
package org.apache.streampark.flink.connector.conf
-import java.util.Properties
-
import org.apache.streampark.common.conf.ConfigOption
import org.apache.streampark.flink.connector.conf
import org.apache.streampark.flink.connector.conf.FailoverStorageType.FailoverStorageType
+import java.util.Properties
+
object ThresholdConfigOption {
- def apply(prefixStr: String, properties: Properties = new Properties): ThresholdConfigOption = new ThresholdConfigOption(prefixStr, properties)
+ def apply(prefixStr: String, properties: Properties = new Properties): ThresholdConfigOption =
+ new ThresholdConfigOption(prefixStr, properties)
}
class ThresholdConfigOption(prefixStr: String, properties: Properties) {
@@ -76,7 +77,8 @@ class ThresholdConfigOption(prefixStr: String, properties: Properties) {
defaultValue = FailoverStorageType.NONE,
handle = k => {
FailoverStorageType.get(properties.getProperty(k))
- })
+ }
+ )
val failoverTable: ConfigOption[String] = ConfigOption(
key = "failover.table",
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/FailoverChecker.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/FailoverChecker.scala
index a906f2e4c..11a30f5f7 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/FailoverChecker.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/FailoverChecker.scala
@@ -17,19 +17,24 @@
package org.apache.streampark.flink.connector.failover
+import org.apache.streampark.common.util.{Logger, ThreadUtils}
+
import java.util.concurrent.{Executors, ScheduledExecutorService, ThreadFactory, TimeUnit}
import scala.collection.JavaConversions._
import scala.collection.mutable.ListBuffer
-import org.apache.streampark.common.util.{Logger, ThreadUtils}
-
case class FailoverChecker(delayTime: Long) extends AutoCloseable with Logger {
val sinkBuffers: ListBuffer[SinkBuffer] = ListBuffer[SinkBuffer]()
val factory: ThreadFactory = ThreadUtils.threadFactory("FailoverChecker")
- val scheduledExecutorService: ScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(factory)
- scheduledExecutorService.scheduleWithFixedDelay(getTask, delayTime, delayTime, TimeUnit.MILLISECONDS)
+ val scheduledExecutorService: ScheduledExecutorService =
+ Executors.newSingleThreadScheduledExecutor(factory)
+ scheduledExecutorService.scheduleWithFixedDelay(
+ getTask,
+ delayTime,
+ delayTime,
+ TimeUnit.MILLISECONDS)
logInfo(s"Build Sink scheduled checker, timeout (microSeconds) = $delayTime")
def addSinkBuffer(buffer: SinkBuffer): Unit = {
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/FailoverWriter.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/FailoverWriter.scala
index 0a352fed2..515583929 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/FailoverWriter.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/FailoverWriter.scala
@@ -17,18 +17,20 @@
package org.apache.streampark.flink.connector.failover
+import org.apache.streampark.common.conf.ConfigConst._
+import org.apache.streampark.common.util._
+import org.apache.streampark.flink.connector.conf.FailoverStorageType._
+
+import org.apache.kafka.clients.producer.{Callback, KafkaProducer, ProducerRecord, RecordMetadata}
+
import java.util._
import java.util.concurrent.locks.ReentrantLock
import scala.collection.JavaConversions._
-import org.apache.kafka.clients.producer.{Callback, KafkaProducer, ProducerRecord, RecordMetadata}
-
-import org.apache.streampark.common.conf.ConfigConst._
-import org.apache.streampark.common.util._
-import org.apache.streampark.flink.connector.conf.FailoverStorageType._
-
-class FailoverWriter(failoverStorage: FailoverStorageType, properties: Properties) extends AutoCloseable with Logger {
+class FailoverWriter(failoverStorage: FailoverStorageType, properties: Properties)
+ extends AutoCloseable
+ with Logger {
private[this] object Lock {
@volatile var initialized = false
@@ -51,13 +53,19 @@ class FailoverWriter(failoverStorage: FailoverStorageType, properties: Propertie
Lock.lock.lock()
if (!Lock.initialized) {
Lock.initialized = true
- properties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer")
- properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer")
+ properties.put(
+ "key.serializer",
+ "org.apache.kafka.common.serialization.StringSerializer")
+ properties.put(
+ "value.serializer",
+ "org.apache.kafka.common.serialization.StringSerializer")
kafkaProducer = new KafkaProducer[String, String](properties)
}
} catch {
case exception: Exception => {
- logError(s"build Failover storageType:KAFKA failed exception ${exception.getStackTrace.mkString("Array(", ", ", ")")}")
+ logError(
+ s"build Failover storageType:KAFKA failed exception ${exception.getStackTrace
+ .mkString("Array(", ", ", ")")}")
throw exception
}
} finally {
@@ -75,13 +83,17 @@ class FailoverWriter(failoverStorage: FailoverStorageType, properties: Propertie
|}
|""".stripMargin
val record = new ProducerRecord[String, String](topic, sendData)
- kafkaProducer.send(
- record,
- new Callback() {
- override def onCompletion(recordMetadata: RecordMetadata, e: Exception): Unit = {
- logInfo(s"Failover successful!! storageType:Kafka,table: $table,size:${request.size}")
+ kafkaProducer
+ .send(
+ record,
+ new Callback() {
+ override def onCompletion(recordMetadata: RecordMetadata, e: Exception): Unit = {
+ logInfo(
+ s"Failover successful!! storageType:Kafka,table: $table,size:${request.size}")
+ }
}
- }).get()
+ )
+ .get()
case MySQL =>
if (!Lock.initialized) {
try {
@@ -90,7 +102,8 @@ class FailoverWriter(failoverStorage: FailoverStorageType, properties: Propertie
Lock.initialized = true
properties.put(KEY_ALIAS, s"failover-$table")
val mysqlConnect = JdbcUtils.getConnection(properties)
- val mysqlTable = mysqlConnect.getMetaData.getTables(null, null, table, Array("TABLE", "VIEW"))
+ val mysqlTable =
+ mysqlConnect.getMetaData.getTables(null, null, table, Array("TABLE", "VIEW"))
if (!mysqlTable.next()) {
JdbcUtils.execute(
mysqlConnect,
@@ -100,21 +113,25 @@ class FailoverWriter(failoverStorage: FailoverStorageType, properties: Propertie
}
} catch {
case exception: Exception =>
- logError(s"build Failover storageType:MySQL failed exception ${exception.getStackTrace}")
+ logError(
+ s"build Failover storageType:MySQL failed exception ${exception.getStackTrace}")
throw exception
} finally {
Lock.lock.unlock()
}
}
val timestamp = System.currentTimeMillis()
- val records = request.records.map(x => {
- val v = cleanUp(x)
- s""" ($v,$timestamp) """.stripMargin
- })
+ val records = request.records.map(
+ x => {
+ val v = cleanUp(x)
+ s""" ($v,$timestamp) """.stripMargin
+ })
val sql = s"INSERT INTO $table(`values`,`timestamp`) VALUES ${records.mkString(",")} "
JdbcUtils.update(sql)(properties)
logInfo(s"Failover successful!! storageType:MySQL,table: $table,size:${request.size}")
- case _ => throw new UnsupportedOperationException(s"[StreamPark] unsupported failover storageType:$failoverStorage")
+ case _ =>
+ throw new UnsupportedOperationException(
+ s"[StreamPark] unsupported failover storageType:$failoverStorage")
}
}
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkBuffer.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkBuffer.scala
index e8bb2829c..531f5c824 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkBuffer.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkBuffer.scala
@@ -17,15 +17,17 @@
package org.apache.streampark.flink.connector.failover
+import org.apache.streampark.common.util.Logger
+
import java.util
import java.util.Collections
import java.util.concurrent.CopyOnWriteArrayList
import scala.collection.JavaConversions._
-import org.apache.streampark.common.util.Logger
-
-case class SinkBuffer(writer: SinkWriter, flushInterval: Long, bufferSize: Int) extends AutoCloseable with Logger {
+case class SinkBuffer(writer: SinkWriter, flushInterval: Long, bufferSize: Int)
+ extends AutoCloseable
+ with Logger {
private var timestamp = 0L
@@ -65,7 +67,8 @@ case class SinkBuffer(writer: SinkWriter, flushInterval: Long, bufferSize: Int)
} else false
}
- private[this] def buildDeepCopy(original: util.List[String]): util.List[String] = Collections.unmodifiableList(new util.ArrayList[String](original))
+ private[this] def buildDeepCopy(original: util.List[String]): util.List[String] =
+ Collections.unmodifiableList(new util.ArrayList[String](original))
override def close(): Unit = if (localValues.nonEmpty) addToQueue()
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkRequest.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkRequest.scala
index 10a01ce50..c1b8b7a7f 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkRequest.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkRequest.scala
@@ -27,27 +27,28 @@ case class SinkRequest(records: util.List[String], var attemptCounter: Int = 0)
def size: Int = records.size()
- private[this] lazy val TABLE_REGEXP = Pattern.compile(
- "(insert\\s+into|update|delete)\\s+(.*?)(\\(|\\s+)",
- Pattern.CASE_INSENSITIVE)
+ private[this] lazy val TABLE_REGEXP =
+ Pattern.compile("(insert\\s+into|update|delete)\\s+(.*?)(\\(|\\s+)", Pattern.CASE_INSENSITIVE)
- private[this] lazy val INSERT_REGEXP = Pattern.compile(
- "^(.*)\\s+(values|value)(.*)",
- Pattern.CASE_INSENSITIVE)
+ private[this] lazy val INSERT_REGEXP =
+ Pattern.compile("^(.*)\\s+(values|value)(.*)", Pattern.CASE_INSENSITIVE)
lazy val sqlStatement: String = {
val matcher = INSERT_REGEXP.matcher(records.head)
if (!matcher.find()) null;
else {
val prefix = matcher.group(1)
- val values = records.map(x => {
- val valueMatcher = INSERT_REGEXP.matcher(x)
- if (valueMatcher.find()) {
- valueMatcher.group(3)
- } else {
- null
- }
- }).mkString(",")
+ val values = records
+ .map(
+ x => {
+ val valueMatcher = INSERT_REGEXP.matcher(x)
+ if (valueMatcher.find()) {
+ valueMatcher.group(3)
+ } else {
+ null
+ }
+ })
+ .mkString(",")
s"$prefix VALUES $values"
}
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/sink/Sink.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/sink/Sink.scala
index 1f31e4891..a830cc011 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/sink/Sink.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/sink/Sink.scala
@@ -21,7 +21,11 @@ import org.apache.flink.streaming.api.datastream.DataStreamSink
trait Sink extends Serializable {
- def afterSink[T](sink: DataStreamSink[T], parallelism: Int, name: String, uid: String): DataStreamSink[T] = {
+ def afterSink[T](
+ sink: DataStreamSink[T],
+ parallelism: Int,
+ name: String,
+ uid: String): DataStreamSink[T] = {
if (parallelism > 0) {
sink.setParallelism(parallelism)
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseHttpConfig.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseHttpConfig.scala
index 0e961a7be..41d8480ce 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseHttpConfig.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseHttpConfig.scala
@@ -17,14 +17,14 @@
package org.apache.streampark.flink.connector.clickhouse.conf
+import org.apache.streampark.common.conf.ConfigConst
+import org.apache.streampark.flink.connector.conf.ThresholdConf
+
import java.util.{Base64, Properties}
import java.util.concurrent.ThreadLocalRandom
import scala.collection.JavaConversions._
-import org.apache.streampark.common.conf.ConfigConst
-import org.apache.streampark.flink.connector.conf.ThresholdConf
-
/**
* Flink sink for Clickhouse database. Powered by Async Http Client.
*
@@ -34,9 +34,11 @@ import org.apache.streampark.flink.connector.conf.ThresholdConf
*/
//---------------------------------------------------------------------------------------
-class ClickHouseHttpConfig(parameters: Properties) extends ThresholdConf(ClickHouseSinkConfigOption.CLICKHOUSE_SINK_PREFIX, parameters) {
+class ClickHouseHttpConfig(parameters: Properties)
+ extends ThresholdConf(ClickHouseSinkConfigOption.CLICKHOUSE_SINK_PREFIX, parameters) {
- @transient val sinkOption: ClickHouseSinkConfigOption = ClickHouseSinkConfigOption(properties = parameters)
+ @transient val sinkOption: ClickHouseSinkConfigOption =
+ ClickHouseSinkConfigOption(properties = parameters)
val user: String = sinkOption.user.get()
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseSinkConfigOption.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseSinkConfigOption.scala
index 39c9a4b84..527951a47 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseSinkConfigOption.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseSinkConfigOption.scala
@@ -17,14 +17,14 @@
package org.apache.streampark.flink.connector.clickhouse.conf
-import java.util.Properties
-
-import scala.collection.JavaConverters._
+import org.apache.streampark.common.conf.ConfigOption
+import org.apache.streampark.common.util.ConfigUtils
import org.asynchttpclient.config.AsyncHttpClientConfigDefaults
-import org.apache.streampark.common.conf.ConfigOption
-import org.apache.streampark.common.util.ConfigUtils
+import java.util.Properties
+
+import scala.collection.JavaConverters._
object ClickHouseSinkConfigOption {
@@ -34,7 +34,9 @@ object ClickHouseSinkConfigOption {
* @param properties
* @return
*/
- def apply(prefixStr: String = CLICKHOUSE_SINK_PREFIX, properties: Properties = new Properties): ClickHouseSinkConfigOption =
+ def apply(
+ prefixStr: String = CLICKHOUSE_SINK_PREFIX,
+ properties: Properties = new Properties): ClickHouseSinkConfigOption =
new ClickHouseSinkConfigOption(prefixStr, properties)
}
@@ -57,18 +59,14 @@ class ClickHouseSinkConfigOption(prefixStr: String, properties: Properties) exte
.filter(_.nonEmpty)
.map(_.replaceAll("\\s+", "").replaceFirst("^http://|^", "http://"))
.toList
- })
+ }
+ )
- val user: ConfigOption[String] = ConfigOption(
- key = "user",
- required = true,
- classType = classOf[String])
+ val user: ConfigOption[String] =
+ ConfigOption(key = "user", required = true, classType = classOf[String])
- val password: ConfigOption[String] = ConfigOption(
- key = "password",
- required = false,
- defaultValue = "",
- classType = classOf[String])
+ val password: ConfigOption[String] =
+ ConfigOption(key = "password", required = false, defaultValue = "", classType = classOf[String])
val database: ConfigOption[String] = ConfigOption(
key = "database",
@@ -100,15 +98,11 @@ class ClickHouseSinkConfigOption(prefixStr: String, properties: Properties) exte
defaultValue = AsyncHttpClientConfigDefaults.defaultMaxConnections(),
classType = classOf[Int])
- val failoverTable: ConfigOption[String] = ConfigOption(
- key = "failover.table",
- required = false,
- classType = classOf[String])
+ val failoverTable: ConfigOption[String] =
+ ConfigOption(key = "failover.table", required = false, classType = classOf[String])
- val jdbcUrl: ConfigOption[String] = ConfigOption(
- key = "jdbcUrl",
- required = false,
- classType = classOf[String])
+ val jdbcUrl: ConfigOption[String] =
+ ConfigOption(key = "jdbcUrl", required = false, classType = classOf[String])
val driverClassName: ConfigOption[String] = ConfigOption(
key = "driverClassName",
@@ -116,11 +110,8 @@ class ClickHouseSinkConfigOption(prefixStr: String, properties: Properties) exte
defaultValue = null,
classType = classOf[String])
- val batchSize: ConfigOption[Int] = ConfigOption(
- key = "batchSize",
- required = false,
- defaultValue = 1,
- classType = classOf[Int])
+ val batchSize: ConfigOption[Int] =
+ ConfigOption(key = "batchSize", required = false, defaultValue = 1, classType = classOf[Int])
val flushInterval: ConfigOption[Long] = ConfigOption(
key = "flushInterval",
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/AsyncClickHouseSinkFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/AsyncClickHouseSinkFunction.scala
index f4d85ff41..14c65d3fd 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/AsyncClickHouseSinkFunction.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/AsyncClickHouseSinkFunction.scala
@@ -17,13 +17,6 @@
package org.apache.streampark.flink.connector.clickhouse.internal
-import java.util.Properties
-
-import scala.util.{Failure, Try}
-
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.streaming.api.functions.sink.RichSinkFunction
-
import org.apache.streampark.common.enums.ApiType
import org.apache.streampark.common.enums.ApiType.ApiType
import org.apache.streampark.common.util.Logger
@@ -33,7 +26,16 @@ import org.apache.streampark.flink.connector.clickhouse.util.ClickhouseConvertUt
import org.apache.streampark.flink.connector.failover.{FailoverChecker, SinkBuffer}
import org.apache.streampark.flink.connector.function.TransformFunction
-class AsyncClickHouseSinkFunction[T](apiType: ApiType = ApiType.scala, properties: Properties) extends RichSinkFunction[T] with Logger {
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction
+
+import java.util.Properties
+
+import scala.util.{Failure, Try}
+
+class AsyncClickHouseSinkFunction[T](apiType: ApiType = ApiType.scala, properties: Properties)
+ extends RichSinkFunction[T]
+ with Logger {
private[this] object Lock {
@volatile var initialized = false
@@ -69,7 +71,8 @@ class AsyncClickHouseSinkFunction[T](apiType: ApiType = ApiType.scala, propertie
clickHouseConf = new ClickHouseHttpConfig(properties)
clickHouseWriter = internal.ClickHouseSinkWriter(clickHouseConf)
failoverChecker = FailoverChecker(clickHouseConf.delayTime)
- sinkBuffer = SinkBuffer(clickHouseWriter, clickHouseConf.delayTime, clickHouseConf.bufferSize)
+ sinkBuffer =
+ SinkBuffer(clickHouseWriter, clickHouseConf.delayTime, clickHouseConf.bufferSize)
failoverChecker.addSinkBuffer(sinkBuffer)
logInfo("AsyncClickHouseSink initialize... ")
}
@@ -80,7 +83,8 @@ class AsyncClickHouseSinkFunction[T](apiType: ApiType = ApiType.scala, propertie
override def invoke(value: T): Unit = {
val sql = (javaSqlFunc, scalaSqlFunc) match {
case (null, null) => convert[T](value)
- case _ => apiType match {
+ case _ =>
+ apiType match {
case ApiType.java => javaSqlFunc.transform(value)
case ApiType.scala => scalaSqlFunc(value)
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkFunction.scala
index b071c9cbd..038f2dff6 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkFunction.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkFunction.scala
@@ -17,6 +17,18 @@
package org.apache.streampark.flink.connector.clickhouse.internal
+import org.apache.streampark.common.enums.ApiType
+import org.apache.streampark.common.enums.ApiType.ApiType
+import org.apache.streampark.common.util.{JdbcUtils, Logger}
+import org.apache.streampark.flink.connector.clickhouse.conf.ClickHouseJdbcConfig
+import org.apache.streampark.flink.connector.clickhouse.util.ClickhouseConvertUtils.convert
+import org.apache.streampark.flink.connector.function.TransformFunction
+
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction}
+import ru.yandex.clickhouse.ClickHouseDataSource
+import ru.yandex.clickhouse.settings.ClickHouseProperties
+
import java.sql.{Connection, Statement}
import java.util
import java.util.Properties
@@ -25,19 +37,9 @@ import java.util.concurrent.atomic.AtomicLong
import scala.collection.JavaConversions._
import scala.util.Try
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction}
-import ru.yandex.clickhouse.ClickHouseDataSource
-import ru.yandex.clickhouse.settings.ClickHouseProperties
-
-import org.apache.streampark.common.enums.ApiType
-import org.apache.streampark.common.enums.ApiType.ApiType
-import org.apache.streampark.common.util.{JdbcUtils, Logger}
-import org.apache.streampark.flink.connector.clickhouse.conf.ClickHouseJdbcConfig
-import org.apache.streampark.flink.connector.clickhouse.util.ClickhouseConvertUtils.convert
-import org.apache.streampark.flink.connector.function.TransformFunction
-
-class ClickHouseSinkFunction[T](apiType: ApiType = ApiType.scala, config: Properties) extends RichSinkFunction[T] with Logger {
+class ClickHouseSinkFunction[T](apiType: ApiType = ApiType.scala, config: Properties)
+ extends RichSinkFunction[T]
+ with Logger {
private var connection: Connection = _
private var statement: Statement = _
var clickHouseConf: ClickHouseJdbcConfig = new ClickHouseJdbcConfig(config)
@@ -78,21 +80,25 @@ class ClickHouseSinkFunction[T](apiType: ApiType = ApiType.scala, config: Proper
case _ => properties.setUser(user)
}
// reflect set all properties...
- clickHouseConf.sinkOption.getInternalConfig().foreach(x => {
- Try(Option(properties.getClass.getDeclaredField(x._1))).getOrElse(None) match {
- case Some(field) =>
- field.setAccessible(true)
- field.getType.getSimpleName match {
- case "String" => field.set(properties, x._2)
- case "int" | "Integer" => field.set(properties, x._2.toInt)
- case "long" | "Long" => field.set(properties, x._2.toLong)
- case "boolean" | "Boolean" => field.set(properties, x._2.toBoolean)
- case _ =>
+ clickHouseConf.sinkOption
+ .getInternalConfig()
+ .foreach(
+ x => {
+ Try(Option(properties.getClass.getDeclaredField(x._1))).getOrElse(None) match {
+ case Some(field) =>
+ field.setAccessible(true)
+ field.getType.getSimpleName match {
+ case "String" => field.set(properties, x._2)
+ case "int" | "Integer" => field.set(properties, x._2.toInt)
+ case "long" | "Long" => field.set(properties, x._2.toLong)
+ case "boolean" | "Boolean" => field.set(properties, x._2.toBoolean)
+ case _ =>
+ }
+ case None =>
+ logWarn(
+ s"ClickHouseProperties config error,property:${x._1} invalid,please see ru.yandex.clickhouse.settings.ClickHouseProperties")
}
- case None =>
- logWarn(s"ClickHouseProperties config error,property:${x._1} invalid,please see ru.yandex.clickhouse.settings.ClickHouseProperties")
- }
- })
+ })
val dataSource = new ClickHouseDataSource(clickHouseConf.jdbcUrl, properties)
connection = dataSource.getConnection
}
@@ -101,7 +107,8 @@ class ClickHouseSinkFunction[T](apiType: ApiType = ApiType.scala, config: Proper
require(connection != null)
val sql = (javaSqlFunc, scalaSqlFunc) match {
case (null, null) => convert[T](value)
- case _ => apiType match {
+ case _ =>
+ apiType match {
case ApiType.java => javaSqlFunc.transform(value)
case ApiType.scala => scalaSqlFunc(value)
}
@@ -109,11 +116,11 @@ class ClickHouseSinkFunction[T](apiType: ApiType = ApiType.scala, config: Proper
batchSize match {
case 1 =>
- Try(connection.prepareStatement(sql).executeUpdate)
- .recover { case e =>
+ Try(connection.prepareStatement(sql).executeUpdate).recover {
+ case e =>
logError(s"ClickHouseSink invoke error: $e")
throw e
- }.get
+ }.get
case batch =>
Try {
sqlValues.add(sql)
@@ -144,7 +151,7 @@ class ClickHouseSinkFunction[T](apiType: ApiType = ApiType.scala, config: Proper
val sql = s"$insertSqlPrefixes $valuesStr"
// clickhouse batch insert return num always 1
val insertNum: Int = connection.prepareStatement(sql).executeUpdate()
- logInfo(s"ClickHouseSink batch successful, execute size : ${insertNum}")
+ logInfo(s"ClickHouseSink batch successful, execute size : $insertNum")
timestamp = System.currentTimeMillis()
} finally {
sqlValues.clear()
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkWriter.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkWriter.scala
index fa2ecbda1..590025895 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkWriter.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkWriter.scala
@@ -17,20 +17,23 @@
package org.apache.streampark.flink.connector.clickhouse.internal
-import java.util.concurrent._
-
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ListBuffer
-
-import org.asynchttpclient.{AsyncHttpClient, DefaultAsyncHttpClientConfig, Dsl}
-
import org.apache.streampark.common.util.{Logger, ThreadUtils}
import org.apache.streampark.flink.connector.clickhouse.conf.ClickHouseHttpConfig
import org.apache.streampark.flink.connector.clickhouse.internal
import org.apache.streampark.flink.connector.failover.{SinkRequest, SinkWriter}
-case class ClickHouseSinkWriter(clickHouseConfig: ClickHouseHttpConfig) extends SinkWriter with Logger {
- private val callbackServiceFactory = ThreadUtils.threadFactory("ClickHouse-writer-callback-executor")
+import org.asynchttpclient.{AsyncHttpClient, DefaultAsyncHttpClientConfig, Dsl}
+
+import java.util.concurrent._
+
+import scala.collection.JavaConversions._
+import scala.collection.mutable.ListBuffer
+
+case class ClickHouseSinkWriter(clickHouseConfig: ClickHouseHttpConfig)
+ extends SinkWriter
+ with Logger {
+ private val callbackServiceFactory =
+ ThreadUtils.threadFactory("ClickHouse-writer-callback-executor")
private val threadFactory: ThreadFactory = ThreadUtils.threadFactory("ClickHouse-writer")
var callbackService: ExecutorService = new ThreadPoolExecutor(
@@ -42,7 +45,8 @@ case class ClickHouseSinkWriter(clickHouseConfig: ClickHouseHttpConfig) extends
callbackServiceFactory)
var tasks: ListBuffer[ClickHouseWriterTask] = ListBuffer[ClickHouseWriterTask]()
- var recordQueue: BlockingQueue[SinkRequest] = new LinkedBlockingQueue[SinkRequest](clickHouseConfig.queueCapacity)
+ var recordQueue: BlockingQueue[SinkRequest] =
+ new LinkedBlockingQueue[SinkRequest](clickHouseConfig.queueCapacity)
var asyncHttpClient: AsyncHttpClient = Dsl.asyncHttpClient(
new DefaultAsyncHttpClientConfig.Builder()
.setRequestTimeout(clickHouseConfig.sinkOption.requestTimeout.get())
@@ -50,10 +54,16 @@ case class ClickHouseSinkWriter(clickHouseConfig: ClickHouseHttpConfig) extends
.setMaxRequestRetry(clickHouseConfig.sinkOption.maxRequestRetry.get())
.setMaxConnections(clickHouseConfig.sinkOption.maxConnections.get())
.build())
- var service: ExecutorService = Executors.newFixedThreadPool(clickHouseConfig.numWriters, threadFactory)
+ var service: ExecutorService =
+ Executors.newFixedThreadPool(clickHouseConfig.numWriters, threadFactory)
for (i <- 0 until clickHouseConfig.numWriters) {
- val task = internal.ClickHouseWriterTask(i, clickHouseConfig, asyncHttpClient, recordQueue, callbackService)
+ val task = internal.ClickHouseWriterTask(
+ i,
+ clickHouseConfig,
+ asyncHttpClient,
+ recordQueue,
+ callbackService)
tasks.add(task)
service.submit(task)
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseWriterTask.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseWriterTask.scala
index 0d1753e0e..a2de0d449 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseWriterTask.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseWriterTask.scala
@@ -17,28 +17,32 @@
package org.apache.streampark.flink.connector.clickhouse.internal
-import java.util.concurrent.{BlockingQueue, ExecutorService, TimeUnit}
-
-import scala.collection.JavaConversions._
-import scala.util.Try
+import org.apache.streampark.common.util.Logger
+import org.apache.streampark.flink.connector.clickhouse.conf.ClickHouseHttpConfig
+import org.apache.streampark.flink.connector.failover.{FailoverWriter, SinkRequest}
import io.netty.handler.codec.http.{HttpHeaderNames, HttpHeaders}
import org.asynchttpclient.{AsyncHttpClient, ListenableFuture, Request, Response}
-import org.apache.streampark.common.util.Logger
-import org.apache.streampark.flink.connector.clickhouse.conf.ClickHouseHttpConfig
-import org.apache.streampark.flink.connector.failover.{FailoverWriter, SinkRequest}
+import java.util.concurrent.{BlockingQueue, ExecutorService, TimeUnit}
+
+import scala.collection.JavaConversions._
+import scala.util.Try
case class ClickHouseWriterTask(
id: Int,
clickHouseConf: ClickHouseHttpConfig,
asyncHttpClient: AsyncHttpClient,
queue: BlockingQueue[SinkRequest],
- callbackService: ExecutorService) extends Runnable with AutoCloseable with Logger {
+ callbackService: ExecutorService)
+ extends Runnable
+ with AutoCloseable
+ with Logger {
@volatile var isWorking = false
- val failoverWriter: FailoverWriter = new FailoverWriter(clickHouseConf.storageType, clickHouseConf.getFailoverConfig)
+ val failoverWriter: FailoverWriter =
+ new FailoverWriter(clickHouseConf.storageType, clickHouseConf.getFailoverConfig)
override def run(): Unit =
try {
@@ -79,22 +83,26 @@ case class ClickHouseWriterTask(
builder.build
}
- def respCallback(whenResponse: ListenableFuture[Response], sinkRequest: SinkRequest): Runnable = new Runnable {
- override def run(): Unit = {
- Try(whenResponse.get()).getOrElse(null) match {
- case null =>
- logError(s"""Error ClickHouseSink executing callback, params = $clickHouseConf,can not get Response. """)
- handleFailedResponse(null, sinkRequest)
- case resp if resp.getStatusCode != 200 =>
- logError(s"Error ClickHouseSink executing callback, params = ${clickHouseConf}, StatusCode = ${resp.getStatusCode} ")
- handleFailedResponse(resp, sinkRequest)
- case _ =>
+ def respCallback(whenResponse: ListenableFuture[Response], sinkRequest: SinkRequest): Runnable =
+ new Runnable {
+ override def run(): Unit = {
+ Try(whenResponse.get()).getOrElse(null) match {
+ case null =>
+ logError(
+ s"""Error ClickHouseSink executing callback, params = $clickHouseConf,can not get Response. """)
+ handleFailedResponse(null, sinkRequest)
+ case resp if resp.getStatusCode != 200 =>
+ logError(
+ s"Error ClickHouseSink executing callback, params = $clickHouseConf, StatusCode = ${resp.getStatusCode} ")
+ handleFailedResponse(resp, sinkRequest)
+ case _ =>
+ }
}
}
- }
/**
- * if send data to ClickHouse Failed, retry maxRetries, if still failed,flush data to failoverStorage
+ * if send data to ClickHouse Failed, retry maxRetries, if still failed,flush data to
+ * failoverStorage
*
* @param response
* @param sinkRequest
@@ -104,10 +112,12 @@ case class ClickHouseWriterTask(
logWarn(
s"""Failed to send data to ClickHouse, cause: limit of attempts is exceeded. ClickHouse response = $response. Ready to flush data to ${clickHouseConf.storageType}""")
failoverWriter.write(sinkRequest)
- logInfo(s"failover Successful, StorageType = ${clickHouseConf.storageType}, size = ${sinkRequest.size}")
+ logInfo(
+ s"failover Successful, StorageType = ${clickHouseConf.storageType}, size = ${sinkRequest.size}")
} else {
sinkRequest.incrementCounter()
- logWarn(s"Next attempt to send data to ClickHouse, table = ${sinkRequest.table}, buffer size = ${sinkRequest.size}, current attempt num = ${sinkRequest.attemptCounter}, max attempt num = ${clickHouseConf.maxRetries}, response = $response")
+ logWarn(
+ s"Next attempt to send data to ClickHouse, table = ${sinkRequest.table}, buffer size = ${sinkRequest.size}, current attempt num = ${sinkRequest.attemptCounter}, max attempt num = ${clickHouseConf.maxRetries}, response = $response")
queue.put(sinkRequest)
}
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/AsyncClickHouseOutputFormat.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/AsyncClickHouseOutputFormat.scala
index 5fb17e691..350b1eb7f 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/AsyncClickHouseOutputFormat.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/AsyncClickHouseOutputFormat.scala
@@ -17,17 +17,19 @@
package org.apache.streampark.flink.connector.clickhouse.sink
-import java.util.Properties
+import org.apache.streampark.common.util.Logger
+import org.apache.streampark.flink.connector.clickhouse.internal.AsyncClickHouseSinkFunction
+import org.apache.streampark.flink.connector.function.TransformFunction
import org.apache.flink.api.common.io.RichOutputFormat
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.configuration.Configuration
-import org.apache.streampark.common.util.Logger
-import org.apache.streampark.flink.connector.clickhouse.internal.AsyncClickHouseSinkFunction
-import org.apache.streampark.flink.connector.function.TransformFunction
+import java.util.Properties
-class AsyncClickHouseOutputFormat[T: TypeInformation](prop: Properties) extends RichOutputFormat[T] with Logger {
+class AsyncClickHouseOutputFormat[T: TypeInformation](prop: Properties)
+ extends RichOutputFormat[T]
+ with Logger {
var sinkFunction: AsyncClickHouseSinkFunction[T] = _
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/ClickHouseOutputFormat.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/ClickHouseOutputFormat.scala
index 2dbec41ee..2a89d5e2f 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/ClickHouseOutputFormat.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/ClickHouseOutputFormat.scala
@@ -17,17 +17,19 @@
package org.apache.streampark.flink.connector.clickhouse.sink
-import java.util.Properties
+import org.apache.streampark.common.util.Logger
+import org.apache.streampark.flink.connector.clickhouse.internal.ClickHouseSinkFunction
+import org.apache.streampark.flink.connector.function.TransformFunction
import org.apache.flink.api.common.io.RichOutputFormat
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.configuration.Configuration
-import org.apache.streampark.common.util.Logger
-import org.apache.streampark.flink.connector.clickhouse.internal.ClickHouseSinkFunction
-import org.apache.streampark.flink.connector.function.TransformFunction
+import java.util.Properties
-class ClickHouseOutputFormat[T: TypeInformation](prop: Properties) extends RichOutputFormat[T] with Logger {
+class ClickHouseOutputFormat[T: TypeInformation](prop: Properties)
+ extends RichOutputFormat[T]
+ with Logger {
var sinkFunction: ClickHouseSinkFunction[T] = _
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/ClickHouseSink.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/ClickHouseSink.scala
index 5f332dec7..c2e5158cb 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/ClickHouseSink.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/ClickHouseSink.scala
@@ -17,20 +17,20 @@
package org.apache.streampark.flink.connector.clickhouse.sink
-import java.util.Properties
-
-import scala.annotation.meta.param
-
-import org.apache.flink.streaming.api.datastream.{DataStream => JavaDataStream}
-import org.apache.flink.streaming.api.datastream.DataStreamSink
-import org.apache.flink.streaming.api.scala.DataStream
-
import org.apache.streampark.common.util._
import org.apache.streampark.flink.connector.clickhouse.internal.{AsyncClickHouseSinkFunction, ClickHouseSinkFunction}
import org.apache.streampark.flink.connector.function.TransformFunction
import org.apache.streampark.flink.connector.sink.Sink
import org.apache.streampark.flink.core.scala.StreamingContext
+import org.apache.flink.streaming.api.datastream.{DataStream => JavaDataStream}
+import org.apache.flink.streaming.api.datastream.DataStreamSink
+import org.apache.flink.streaming.api.scala.DataStream
+
+import java.util.Properties
+
+import scala.annotation.meta.param
+
object ClickHouseSink {
/**
@@ -47,7 +47,8 @@ object ClickHouseSink {
property: Properties = new Properties(),
parallelism: Int = 0,
name: String = null,
- uid: String = null)(implicit ctx: StreamingContext): ClickHouseSink = new ClickHouseSink(ctx, property, parallelism, name, uid)
+ uid: String = null)(implicit ctx: StreamingContext): ClickHouseSink =
+ new ClickHouseSink(ctx, property, parallelism, name, uid)
}
@@ -56,7 +57,9 @@ class ClickHouseSink(
property: Properties = new Properties(),
parallelism: Int = 0,
name: String = null,
- uid: String = null)(implicit alias: String = "") extends Sink with Logger {
+ uid: String = null)(implicit alias: String = "")
+ extends Sink
+ with Logger {
val prop = ctx.parameter.getProperties
@@ -73,12 +76,14 @@ class ClickHouseSink(
/**
* asynchronous Write
*
- * @param scala stream
+ * @param scala
+ * stream
* @param toSQLFn
* @tparam T
* @return
*/
- def asyncSink[T](stream: DataStream[T])(implicit toSQLFn: T => String = null): DataStreamSink[T] = {
+ def asyncSink[T](stream: DataStream[T])(implicit
+ toSQLFn: T => String = null): DataStreamSink[T] = {
require(stream != null, () => s"sink Stream must not null")
val sinkFun = new AsyncClickHouseSinkFunction[T](prop, toSQLFn)
val sink = stream.addSink(sinkFun)
@@ -88,12 +93,15 @@ class ClickHouseSink(
/**
* asynchronous Write
*
- * @param java stream
+ * @param java
+ * stream
* @param toSQLFn
* @tparam T
* @return
*/
- def asyncSink[T](stream: JavaDataStream[T], toSQLFn: TransformFunction[T, String]): DataStreamSink[T] = {
+ def asyncSink[T](
+ stream: JavaDataStream[T],
+ toSQLFn: TransformFunction[T, String]): DataStreamSink[T] = {
require(stream != null, () => s"sink Stream must not null")
val sinkFun = new AsyncClickHouseSinkFunction[T](prop, toSQLFn)
val sink = stream.addSink(sinkFun)
@@ -112,12 +120,14 @@ class ClickHouseSink(
/**
* synchronous Write
*
- * @param scala stream
+ * @param scala
+ * stream
* @param toSQLFn
* @tparam T
* @return
*/
- def jdbcSink[T](stream: DataStream[T])(implicit toSQLFn: T => String = null): DataStreamSink[T] = {
+ def jdbcSink[T](stream: DataStream[T])(implicit
+ toSQLFn: T => String = null): DataStreamSink[T] = {
require(stream != null, () => s"sink Stream must not null")
val sinkFun = new ClickHouseSinkFunction[T](prop, toSQLFn)
val sink = stream.addSink(sinkFun)
@@ -127,12 +137,15 @@ class ClickHouseSink(
/**
* synchronous Write
*
- * @param java stream
+ * @param java
+ * stream
* @param sqlFromFn
* @tparam T
* @return
*/
- def jdbcSink[T](stream: JavaDataStream[T], sqlFromFn: TransformFunction[T, String]): DataStreamSink[T] = {
+ def jdbcSink[T](
+ stream: JavaDataStream[T],
+ sqlFromFn: TransformFunction[T, String]): DataStreamSink[T] = {
require(stream != null, () => s"sink Stream must not null")
val sinkFun = new ClickHouseSinkFunction[T](prop, sqlFromFn)
val sink = stream.addSink(sinkFun)
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/util/ClickhouseConvertUtils.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/util/ClickhouseConvertUtils.scala
index 26acf7f23..f4ef102c6 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/util/ClickhouseConvertUtils.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/util/ClickhouseConvertUtils.scala
@@ -22,14 +22,15 @@ object ClickhouseConvertUtils {
def convert[T](value: T): String = {
val buffer = new StringBuilder("(")
val fields = value.getClass.getDeclaredFields
- fields.foreach(f => {
- f.setAccessible(true)
- val v = f.get(value)
- f.getType.getSimpleName match {
- case "String" => buffer.append(s""""$v",""".stripMargin)
- case _ => buffer.append(s"""$v,""".stripMargin)
- }
- })
+ fields.foreach(
+ f => {
+ f.setAccessible(true)
+ val v = f.get(value)
+ f.getType.getSimpleName match {
+ case "String" => buffer.append(s""""$v",""".stripMargin)
+ case _ => buffer.append(s"""$v,""".stripMargin)
+ }
+ })
buffer.toString().replaceFirst(",$", ")")
}
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/scala/org/apache/streampark/connector/doris/conf/DorisConfig.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/scala/org/apache/streampark/connector/doris/conf/DorisConfig.scala
index 5a767ab97..a38f2cee3 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/scala/org/apache/streampark/connector/doris/conf/DorisConfig.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/scala/org/apache/streampark/connector/doris/conf/DorisConfig.scala
@@ -16,12 +16,12 @@
*/
package org.apache.streampark.connector.doris.conf
+import org.apache.streampark.common.conf.ConfigConst
+
import java.util.Properties
import scala.collection.JavaConversions._
-import org.apache.streampark.common.conf.ConfigConst
-
object DorisConfig {
val CSV = "csv"
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/scala/org/apache/streampark/connector/doris/conf/DorisSinkConfigOption.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/scala/org/apache/streampark/connector/doris/conf/DorisSinkConfigOption.scala
index d562a0c53..a4940df84 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/scala/org/apache/streampark/connector/doris/conf/DorisSinkConfigOption.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/scala/org/apache/streampark/connector/doris/conf/DorisSinkConfigOption.scala
@@ -17,17 +17,19 @@
package org.apache.streampark.connector.doris.conf
+import org.apache.streampark.common.conf.ConfigOption
+import org.apache.streampark.common.util.ConfigUtils
+
import java.util.Properties
import scala.collection.JavaConverters._
-import org.apache.streampark.common.conf.ConfigOption
-import org.apache.streampark.common.util.ConfigUtils
-
object DorisSinkConfigOption {
val DORIS_SINK_PREFIX = "doris.sink"
- def apply(prefixStr: String = DORIS_SINK_PREFIX, properties: Properties = new Properties): DorisSinkConfigOption =
+ def apply(
+ prefixStr: String = DORIS_SINK_PREFIX,
+ properties: Properties = new Properties): DorisSinkConfigOption =
new DorisSinkConfigOption(prefixStr, properties)
}
@@ -49,12 +51,11 @@ class DorisSinkConfigOption(prefixStr: String, properties: Properties) extends S
.filter(_.nonEmpty)
.map(_.replaceAll("\\s+", "").replaceFirst("^http://|^", "http://"))
.toList
- })
+ }
+ )
- val user: ConfigOption[String] = ConfigOption(
- key = "user",
- required = true,
- classType = classOf[String])
+ val user: ConfigOption[String] =
+ ConfigOption(key = "user", required = true, classType = classOf[String])
val loadFormat: ConfigOption[String] = ConfigOption(
key = "loadFormat",
@@ -62,22 +63,14 @@ class DorisSinkConfigOption(prefixStr: String, properties: Properties) extends S
defaultValue = "csv",
classType = classOf[String])
- val password: ConfigOption[String] = ConfigOption(
- key = "password",
- required = true,
- classType = classOf[String])
+ val password: ConfigOption[String] =
+ ConfigOption(key = "password", required = true, classType = classOf[String])
- val database: ConfigOption[String] = ConfigOption(
- key = "database",
- required = false,
- defaultValue = "",
- classType = classOf[String])
+ val database: ConfigOption[String] =
+ ConfigOption(key = "database", required = false, defaultValue = "", classType = classOf[String])
- val table: ConfigOption[String] = ConfigOption(
- key = "table",
- required = false,
- defaultValue = "",
- classType = classOf[String])
+ val table: ConfigOption[String] =
+ ConfigOption(key = "table", required = false, defaultValue = "", classType = classOf[String])
val sinkOfferTimeout: ConfigOption[Long] = ConfigOption(
key = "sinkOfferTimeout",
@@ -121,11 +114,8 @@ class DorisSinkConfigOption(prefixStr: String, properties: Properties) extends S
defaultValue = -1,
classType = classOf[Int])
- val maxRow: ConfigOption[Int] = ConfigOption(
- key = "maxRow",
- required = false,
- defaultValue = 100000,
- classType = classOf[Int])
+ val maxRow: ConfigOption[Int] =
+ ConfigOption(key = "maxRow", required = false, defaultValue = 100000, classType = classOf[Int])
val maxBytes: ConfigOption[Int] = ConfigOption(
key = "maxRow",
@@ -133,11 +123,8 @@ class DorisSinkConfigOption(prefixStr: String, properties: Properties) extends S
defaultValue = 94371840,
classType = classOf[Int])
- val maxRetries: ConfigOption[Int] = ConfigOption(
- key = "maxRetries",
- required = false,
- defaultValue = 1,
- classType = classOf[Int])
+ val maxRetries: ConfigOption[Int] =
+ ConfigOption(key = "maxRetries", required = false, defaultValue = 1, classType = classOf[Int])
val lablePrefix: ConfigOption[String] = ConfigOption(
key = "lablePrefix",
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/conf/ESSinkConfigOption.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/conf/ESSinkConfigOption.scala
index 818c9626d..0fbd4868d 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/conf/ESSinkConfigOption.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/conf/ESSinkConfigOption.scala
@@ -17,14 +17,14 @@
package org.apache.streampark.flink.connector.elasticsearch5.conf
+import org.apache.streampark.common.conf.ConfigOption
+import org.apache.streampark.common.util.ConfigUtils
+
import java.net.InetSocketAddress
import java.util.{Map => JavaMap, Properties}
import scala.collection.JavaConverters._
-import org.apache.streampark.common.conf.ConfigOption
-import org.apache.streampark.common.util.ConfigUtils
-
object ESSinkConfigOption {
val ES_SINK_PREFIX = "es.sink"
@@ -32,7 +32,10 @@ object ESSinkConfigOption {
* @param properties
* @return
*/
- def apply(prefixStr: String = ES_SINK_PREFIX, properties: Properties = new Properties): ESSinkConfigOption = new ESSinkConfigOption(prefixStr, properties)
+ def apply(
+ prefixStr: String = ES_SINK_PREFIX,
+ properties: Properties = new Properties): ESSinkConfigOption =
+ new ESSinkConfigOption(prefixStr, properties)
}
@@ -55,11 +58,16 @@ class ESSinkConfigOption(prefixStr: String, properties: Properties) extends Seri
required = true,
classType = classOf[Array[InetSocketAddress]],
handle = key =>
- properties.getProperty(key).split(SIGN_COMMA).map(x => {
- x.split(SIGN_COLON) match {
- case Array(host, port) => new InetSocketAddress(host, port.toInt)
- }
- }))
+ properties
+ .getProperty(key)
+ .split(SIGN_COMMA)
+ .map(
+ x => {
+ x.split(SIGN_COLON) match {
+ case Array(host, port) => new InetSocketAddress(host, port.toInt)
+ }
+ })
+ )
def getInternalConfig(): JavaMap[String, String] = {
ConfigUtils.getConf(prop.asScala.asJava, prefix)(alias = "").asScala.asJava
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/internal/ESSinkFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/internal/ESSinkFunction.scala
index 65d2185d3..4b23a9cd6 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/internal/ESSinkFunction.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/internal/ESSinkFunction.scala
@@ -17,6 +17,11 @@
package org.apache.streampark.flink.connector.elasticsearch5.internal
+import org.apache.streampark.common.enums.ApiType
+import org.apache.streampark.common.enums.ApiType.ApiType
+import org.apache.streampark.common.util.Logger
+import org.apache.streampark.flink.connector.function.TransformFunction
+
import org.apache.flink.api.common.functions.RuntimeContext
import org.apache.flink.streaming.connectors.elasticsearch.{ElasticsearchSinkFunction, RequestIndexer}
import org.elasticsearch.action.ActionRequest
@@ -24,12 +29,9 @@ import org.elasticsearch.action.delete.DeleteRequest
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.update.UpdateRequest
-import org.apache.streampark.common.enums.ApiType
-import org.apache.streampark.common.enums.ApiType.ApiType
-import org.apache.streampark.common.util.Logger
-import org.apache.streampark.flink.connector.function.TransformFunction
-
-class ESSinkFunction[T](apiType: ApiType = ApiType.scala) extends ElasticsearchSinkFunction[T] with Logger {
+class ESSinkFunction[T](apiType: ApiType = ApiType.scala)
+ extends ElasticsearchSinkFunction[T]
+ with Logger {
private[this] var scalaFunc: T => ActionRequest = _
private[this] var javaFunc: TransformFunction[T, ActionRequest] = _
@@ -51,14 +53,21 @@ class ESSinkFunction[T](apiType: ApiType = ApiType.scala) extends ElasticsearchS
case ApiType.scala => scalaFunc(element)
}
- override def process(element: T, runtimeContext: RuntimeContext, requestIndexer: RequestIndexer): Unit = {
+ override def process(
+ element: T,
+ runtimeContext: RuntimeContext,
+ requestIndexer: RequestIndexer): Unit = {
val request: ActionRequest = createIndexRequest(element)
request match {
- case indexRequest if indexRequest.isInstanceOf[IndexRequest] => requestIndexer.add(indexRequest.asInstanceOf[IndexRequest])
- case deleteRequest if deleteRequest.isInstanceOf[DeleteRequest] => requestIndexer.add(deleteRequest.asInstanceOf[DeleteRequest])
- case updateRequest if updateRequest.isInstanceOf[UpdateRequest] => requestIndexer.add(updateRequest.asInstanceOf[UpdateRequest])
+ case indexRequest if indexRequest.isInstanceOf[IndexRequest] =>
+ requestIndexer.add(indexRequest.asInstanceOf[IndexRequest])
+ case deleteRequest if deleteRequest.isInstanceOf[DeleteRequest] =>
+ requestIndexer.add(deleteRequest.asInstanceOf[DeleteRequest])
+ case updateRequest if updateRequest.isInstanceOf[UpdateRequest] =>
+ requestIndexer.add(updateRequest.asInstanceOf[UpdateRequest])
case _ =>
- logError("ElasticsearchSinkFunction add ActionRequest is deprecated, please use IndexRequest|DeleteRequest|UpdateRequest ")
+ logError(
+ "ElasticsearchSinkFunction add ActionRequest is deprecated, please use IndexRequest|DeleteRequest|UpdateRequest ")
requestIndexer.add(request)
}
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/sink/ES5Sink.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/sink/ES5Sink.scala
index ca1e26161..ef658778a 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/sink/ES5Sink.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/sink/ES5Sink.scala
@@ -17,10 +17,12 @@
package org.apache.streampark.flink.connector.elasticsearch5.sink
-import java.util.{Map => JavaMap, Properties}
-
-import scala.annotation.meta.param
-import scala.collection.JavaConversions._
+import org.apache.streampark.common.util.{Logger, Utils}
+import org.apache.streampark.flink.connector.elasticsearch5.conf.ESConfig
+import org.apache.streampark.flink.connector.elasticsearch5.internal.ESSinkFunction
+import org.apache.streampark.flink.connector.function.TransformFunction
+import org.apache.streampark.flink.connector.sink.Sink
+import org.apache.streampark.flink.core.scala.StreamingContext
import org.apache.flink.streaming.api.datastream.{DataStream => JavaDataStream, DataStreamSink}
import org.apache.flink.streaming.api.scala.DataStream
@@ -29,12 +31,10 @@ import org.apache.flink.streaming.connectors.elasticsearch.util.RetryRejectedExe
import org.apache.flink.streaming.connectors.elasticsearch5.ElasticsearchSink
import org.elasticsearch.action.ActionRequest
-import org.apache.streampark.common.util.{Logger, Utils}
-import org.apache.streampark.flink.connector.elasticsearch5.conf.ESConfig
-import org.apache.streampark.flink.connector.elasticsearch5.internal.ESSinkFunction
-import org.apache.streampark.flink.connector.function.TransformFunction
-import org.apache.streampark.flink.connector.sink.Sink
-import org.apache.streampark.flink.core.scala.StreamingContext
+import java.util.{Map => JavaMap, Properties}
+
+import scala.annotation.meta.param
+import scala.collection.JavaConversions._
object ES5Sink {
@@ -43,7 +43,8 @@ object ES5Sink {
property: Properties = new Properties(),
parallelism: Int = 0,
name: String = null,
- uid: String = null)(implicit ctx: StreamingContext): ES5Sink = new ES5Sink(ctx, property, parallelism, name, uid)
+ uid: String = null)(implicit ctx: StreamingContext): ES5Sink =
+ new ES5Sink(ctx, property, parallelism, name, uid)
}
@@ -53,7 +54,9 @@ class ES5Sink(
parallelism: Int = 0,
name: String = null,
uid: String = null,
- alias: String = "") extends Sink with Logger {
+ alias: String = "")
+ extends Sink
+ with Logger {
val prop: Properties = ctx.parameter.getProperties
@@ -72,7 +75,8 @@ class ES5Sink(
f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = {
require(stream != null, () => s"sink Stream must not null")
require(f != null, () => s"es pocess element func must not null")
- val esSink: ElasticsearchSink[T] = new ElasticsearchSink(userConfig, config.host, new ESSinkFunction(f), failureHandler)
+ val esSink: ElasticsearchSink[T] =
+ new ElasticsearchSink(userConfig, config.host, new ESSinkFunction(f), failureHandler)
if (config.disableFlushOnCheckpoint) {
esSink.disableFlushOnCheckpoint()
}
@@ -87,7 +91,8 @@ class ES5Sink(
f: T => ActionRequest): DataStreamSink[T] = {
require(stream != null, () => s"sink Stream must not null")
require(f != null, () => s"es pocess element fun must not null")
- val esSink: ElasticsearchSink[T] = new ElasticsearchSink(userConfig, config.host, new ESSinkFunction(f), failureHandler)
+ val esSink: ElasticsearchSink[T] =
+ new ElasticsearchSink(userConfig, config.host, new ESSinkFunction(f), failureHandler)
if (config.disableFlushOnCheckpoint) {
esSink.disableFlushOnCheckpoint()
}
@@ -107,7 +112,8 @@ class ES5Sink(
def sink[T](
userConfig: JavaMap[String, String],
stream: DataStream[T],
- failureHandler: ActionRequestFailureHandler = new RetryRejectedExecutionFailureHandler)(implicit f: T => ActionRequest): DataStreamSink[T] = {
+ failureHandler: ActionRequestFailureHandler = new RetryRejectedExecutionFailureHandler)(
+ implicit f: T => ActionRequest): DataStreamSink[T] = {
process(userConfig, stream, failureHandler, f)
}
@@ -119,7 +125,10 @@ class ES5Sink(
process(userConfig, stream, failureHandler, f)
}
- def sink[T](userConfig: JavaMap[String, String], stream: JavaDataStream[T], f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = {
+ def sink[T](
+ userConfig: JavaMap[String, String],
+ stream: JavaDataStream[T],
+ f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = {
process(userConfig, stream, new RetryRejectedExecutionFailureHandler, f)
}
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/util/ElasticsearchUtils.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/util/ElasticsearchUtils.scala
index eeadb3e5a..f05a59daf 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/util/ElasticsearchUtils.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/util/ElasticsearchUtils.scala
@@ -23,15 +23,17 @@ import org.elasticsearch.common.xcontent.XContentType
object ElasticsearchUtils {
- def indexRequest(index: String, indexType: String, id: String, source: String)(implicit xContentType: XContentType = XContentType.JSON): IndexRequest = {
+ def indexRequest(index: String, indexType: String, id: String, source: String)(implicit
+ xContentType: XContentType = XContentType.JSON): IndexRequest = {
require(source != null, "indexRequest error:source can not be null...")
require(xContentType != null, "indexRequest error:xContentType can not be null...")
val indexReq = new IndexRequest(index, indexType, id)
val mapping = List("source" -> new BytesArray(source), "contentType" -> xContentType)
- mapping.foreach { x =>
- val field = indexReq.getClass.getDeclaredField(x._1)
- field.setAccessible(true)
- field.set(indexReq, x._2)
+ mapping.foreach {
+ x =>
+ val field = indexReq.getClass.getDeclaredField(x._1)
+ field.setAccessible(true)
+ field.set(indexReq, x._2)
}
indexReq
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/bean/RestClientFactoryImpl.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/bean/RestClientFactoryImpl.scala
index 3394342a4..a5e23c89c 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/bean/RestClientFactoryImpl.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/bean/RestClientFactoryImpl.scala
@@ -17,6 +17,9 @@
package org.apache.streampark.flink.connector.elasticsearch6.bean
+import org.apache.streampark.common.util.Logger
+import org.apache.streampark.flink.connector.elasticsearch6.conf.ES6Config
+
import org.apache.flink.streaming.connectors.elasticsearch6.RestClientFactory
import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials}
import org.apache.http.client.CredentialsProvider
@@ -26,9 +29,6 @@ import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.apache.http.message.BasicHeader
import org.elasticsearch.client.RestClientBuilder
-import org.apache.streampark.common.util.Logger
-import org.apache.streampark.flink.connector.elasticsearch6.conf.ES6Config
-
class RestClientFactoryImpl(val config: ES6Config) extends RestClientFactory with Logger {
override def configureRestClientBuilder(restClientBuilder: RestClientBuilder): Unit = {
// httpClientConfigCallback and requestConfigCallback........
@@ -38,17 +38,21 @@ class RestClientFactoryImpl(val config: ES6Config) extends RestClientFactory wit
// userName,password must be all set,or all not set..
require(
(userName != null && password != null) || (userName == null && password == null),
- "[StreamPark] elasticsearch auth info error,userName,password must be all set,or all not set.")
+ "[StreamPark] elasticsearch auth info error,userName,password must be all set,or all not set."
+ )
val credentialsProvider = (userName, password) match {
case (null, null) => null
case _ =>
val credentialsProvider: CredentialsProvider = new BasicCredentialsProvider()
- credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password))
+ credentialsProvider.setCredentials(
+ AuthScope.ANY,
+ new UsernamePasswordCredentials(userName, password))
credentialsProvider
}
val httpClientConfigCallback = new RestClientBuilder.HttpClientConfigCallback {
- override def customizeHttpClient(httpClientBuilder: HttpAsyncClientBuilder): HttpAsyncClientBuilder = {
+ override def customizeHttpClient(
+ httpClientBuilder: HttpAsyncClientBuilder): HttpAsyncClientBuilder = {
if (credentialsProvider != null) {
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)
logInfo("elasticsearch auth by userName,password...")
@@ -59,7 +63,8 @@ class RestClientFactoryImpl(val config: ES6Config) extends RestClientFactory wit
}
val requestConfigCallback = new RestClientBuilder.RequestConfigCallback {
- override def customizeRequestConfig(requestConfigBuilder: RequestConfig.Builder): RequestConfig.Builder = {
+ override def customizeRequestConfig(
+ requestConfigBuilder: RequestConfig.Builder): RequestConfig.Builder = {
if (credentialsProvider != null) {
requestConfigBuilder.setAuthenticationEnabled(true)
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/conf/ES6Config.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/conf/ES6Config.scala
index 2510538b4..3d9192190 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/conf/ES6Config.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/conf/ES6Config.scala
@@ -17,10 +17,10 @@
package org.apache.streampark.flink.connector.elasticsearch6.conf
-import java.util.Properties
-
import org.apache.http.HttpHost
+import java.util.Properties
+
class ES6Config(parameters: Properties) extends Serializable {
val sinkOption: ESSinkConfigOption = ESSinkConfigOption(properties = parameters)
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/conf/ES6SinkConfigOption.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/conf/ES6SinkConfigOption.scala
index c2fba2c36..c00af5100 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/conf/ES6SinkConfigOption.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/conf/ES6SinkConfigOption.scala
@@ -17,16 +17,16 @@
package org.apache.streampark.flink.connector.elasticsearch6.conf
+import org.apache.streampark.common.conf.ConfigOption
+import org.apache.streampark.common.util.ConfigUtils
+
+import org.apache.http.HttpHost
+
import java.net.InetSocketAddress
import java.util.{Map => JavaMap, Properties}
import scala.collection.JavaConverters._
-import org.apache.http.HttpHost
-
-import org.apache.streampark.common.conf.ConfigOption
-import org.apache.streampark.common.util.ConfigUtils
-
object ESSinkConfigOption {
val ES_SINK_PREFIX = "es.sink"
@@ -35,7 +35,10 @@ object ESSinkConfigOption {
* @param properties
* @return
*/
- def apply(prefixStr: String = ES_SINK_PREFIX, properties: Properties = new Properties): ESSinkConfigOption = new ESSinkConfigOption(prefixStr, properties)
+ def apply(
+ prefixStr: String = ES_SINK_PREFIX,
+ properties: Properties = new Properties): ESSinkConfigOption =
+ new ESSinkConfigOption(prefixStr, properties)
}
@@ -58,11 +61,16 @@ class ESSinkConfigOption(prefixStr: String, properties: Properties) extends Seri
required = true,
classType = classOf[Array[InetSocketAddress]],
handle = key =>
- properties.getProperty(key).split(SIGN_COMMA).map(x => {
- x.split(SIGN_COLON) match {
- case Array(host, port) => new HttpHost(host, port.toInt)
- }
- }))
+ properties
+ .getProperty(key)
+ .split(SIGN_COMMA)
+ .map(
+ x => {
+ x.split(SIGN_COLON) match {
+ case Array(host, port) => new HttpHost(host, port.toInt)
+ }
+ })
+ )
val userName: ConfigOption[String] = ConfigOption(
key = "es.auth.user",
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/internal/ESSinkFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/internal/ESSinkFunction.scala
index 86f2248f4..af2c7afa2 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/internal/ESSinkFunction.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/internal/ESSinkFunction.scala
@@ -17,6 +17,11 @@
package org.apache.streampark.flink.connector.elasticsearch6.internal
+import org.apache.streampark.common.enums.ApiType
+import org.apache.streampark.common.enums.ApiType.ApiType
+import org.apache.streampark.common.util.Logger
+import org.apache.streampark.flink.connector.function.TransformFunction
+
import org.apache.flink.api.common.functions.RuntimeContext
import org.apache.flink.streaming.connectors.elasticsearch.{ElasticsearchSinkFunction, RequestIndexer}
import org.elasticsearch.action.ActionRequest
@@ -24,12 +29,9 @@ import org.elasticsearch.action.delete.DeleteRequest
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.update.UpdateRequest
-import org.apache.streampark.common.enums.ApiType
-import org.apache.streampark.common.enums.ApiType.ApiType
-import org.apache.streampark.common.util.Logger
-import org.apache.streampark.flink.connector.function.TransformFunction
-
-class ESSinkFunction[T](apiType: ApiType = ApiType.scala) extends ElasticsearchSinkFunction[T] with Logger {
+class ESSinkFunction[T](apiType: ApiType = ApiType.scala)
+ extends ElasticsearchSinkFunction[T]
+ with Logger {
private[this] var scalaFunc: T => ActionRequest = _
private[this] var javaFunc: TransformFunction[T, ActionRequest] = _
@@ -50,14 +52,21 @@ class ESSinkFunction[T](apiType: ApiType = ApiType.scala) extends ElasticsearchS
case ApiType.scala => scalaFunc(element)
}
- override def process(element: T, runtimeContext: RuntimeContext, requestIndexer: RequestIndexer): Unit = {
+ override def process(
+ element: T,
+ runtimeContext: RuntimeContext,
+ requestIndexer: RequestIndexer): Unit = {
val request: ActionRequest = createIndexRequest(element)
request match {
- case indexRequest if indexRequest.isInstanceOf[IndexRequest] => requestIndexer.add(indexRequest.asInstanceOf[IndexRequest])
- case deleteRequest if deleteRequest.isInstanceOf[DeleteRequest] => requestIndexer.add(deleteRequest.asInstanceOf[DeleteRequest])
- case updateRequest if updateRequest.isInstanceOf[UpdateRequest] => requestIndexer.add(updateRequest.asInstanceOf[UpdateRequest])
+ case indexRequest if indexRequest.isInstanceOf[IndexRequest] =>
+ requestIndexer.add(indexRequest.asInstanceOf[IndexRequest])
+ case deleteRequest if deleteRequest.isInstanceOf[DeleteRequest] =>
+ requestIndexer.add(deleteRequest.asInstanceOf[DeleteRequest])
+ case updateRequest if updateRequest.isInstanceOf[UpdateRequest] =>
+ requestIndexer.add(updateRequest.asInstanceOf[UpdateRequest])
case _ =>
- logError("ElasticsearchSinkFunction add ActionRequest is deprecated, please use IndexRequest|DeleteRequest|UpdateRequest ")
+ logError(
+ "ElasticsearchSinkFunction add ActionRequest is deprecated, please use IndexRequest|DeleteRequest|UpdateRequest ")
requestIndexer.add(request)
}
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/sink/ES6Sink.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/sink/ES6Sink.scala
index c3708f9b3..8c898089d 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/sink/ES6Sink.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/sink/ES6Sink.scala
@@ -17,10 +17,13 @@
package org.apache.streampark.flink.connector.elasticsearch6.sink
-import java.util.Properties
-
-import scala.annotation.meta.param
-import scala.collection.JavaConversions._
+import org.apache.streampark.common.util.{Logger, Utils}
+import org.apache.streampark.flink.connector.elasticsearch6.bean.RestClientFactoryImpl
+import org.apache.streampark.flink.connector.elasticsearch6.conf.ES6Config
+import org.apache.streampark.flink.connector.elasticsearch6.internal.ESSinkFunction
+import org.apache.streampark.flink.connector.function.TransformFunction
+import org.apache.streampark.flink.connector.sink.Sink
+import org.apache.streampark.flink.core.scala.StreamingContext
import org.apache.flink.streaming.api.datastream.{DataStream => JavaDataStream, DataStreamSink}
import org.apache.flink.streaming.api.scala.DataStream
@@ -30,13 +33,10 @@ import org.apache.flink.streaming.connectors.elasticsearch.util.RetryRejectedExe
import org.apache.flink.streaming.connectors.elasticsearch6.{ElasticsearchSink, RestClientFactory}
import org.elasticsearch.action.ActionRequest
-import org.apache.streampark.common.util.{Logger, Utils}
-import org.apache.streampark.flink.connector.elasticsearch6.bean.RestClientFactoryImpl
-import org.apache.streampark.flink.connector.elasticsearch6.conf.ES6Config
-import org.apache.streampark.flink.connector.elasticsearch6.internal.ESSinkFunction
-import org.apache.streampark.flink.connector.function.TransformFunction
-import org.apache.streampark.flink.connector.sink.Sink
-import org.apache.streampark.flink.core.scala.StreamingContext
+import java.util.Properties
+
+import scala.annotation.meta.param
+import scala.collection.JavaConversions._
object ES6Sink {
@@ -45,12 +45,19 @@ object ES6Sink {
property: Properties = new Properties(),
parallelism: Int = 0,
name: String = null,
- uid: String = null)(implicit ctx: StreamingContext): ES6Sink = new ES6Sink(ctx, property, parallelism, name, uid)
+ uid: String = null)(implicit ctx: StreamingContext): ES6Sink =
+ new ES6Sink(ctx, property, parallelism, name, uid)
}
-class ES6Sink(@(transient @param) ctx: StreamingContext, property: Properties = new Properties(), parallelism: Int = 0, name: String = null, uid: String = null)
- extends Sink with Logger {
+class ES6Sink(
+ @(transient @param) ctx: StreamingContext,
+ property: Properties = new Properties(),
+ parallelism: Int = 0,
+ name: String = null,
+ uid: String = null)
+ extends Sink
+ with Logger {
val prop: Properties = ctx.parameter.getProperties
@@ -124,11 +131,14 @@ class ES6Sink(@(transient @param) ctx: StreamingContext, property: Properties =
// parameter of sink.es.bulk.flush.backoff.enable
case (CONFIG_KEY_BULK_FLUSH_BACKOFF_ENABLE, v) => sinkBuilder.setBulkFlushBackoff(v.toBoolean)
// parameter of sink.es.bulk.flush.backoff.type value of [ CONSTANT or EXPONENTIAL ]
- case (CONFIG_KEY_BULK_FLUSH_BACKOFF_TYPE, v) => sinkBuilder.setBulkFlushBackoffType(FlushBackoffType.valueOf(v))
+ case (CONFIG_KEY_BULK_FLUSH_BACKOFF_TYPE, v) =>
+ sinkBuilder.setBulkFlushBackoffType(FlushBackoffType.valueOf(v))
// parameter of sink.es.bulk.flush.backoff.retries
- case (CONFIG_KEY_BULK_FLUSH_BACKOFF_RETRIES, v) => sinkBuilder.setBulkFlushBackoffRetries(v.toInt)
+ case (CONFIG_KEY_BULK_FLUSH_BACKOFF_RETRIES, v) =>
+ sinkBuilder.setBulkFlushBackoffRetries(v.toInt)
// parameter of sink.es.bulk.flush.backoff.delay
- case (CONFIG_KEY_BULK_FLUSH_BACKOFF_DELAY, v) => sinkBuilder.setBulkFlushBackoffDelay(v.toLong)
+ case (CONFIG_KEY_BULK_FLUSH_BACKOFF_DELAY, v) =>
+ sinkBuilder.setBulkFlushBackoffDelay(v.toLong)
// other...
case _ =>
}
@@ -149,7 +159,8 @@ class ES6Sink(@(transient @param) ctx: StreamingContext, property: Properties =
def sink[T](
stream: DataStream[T],
restClientFactory: Option[RestClientFactory] = None,
- failureHandler: ActionRequestFailureHandler = new RetryRejectedExecutionFailureHandler)(implicit f: T => ActionRequest): DataStreamSink[T] = {
+ failureHandler: ActionRequestFailureHandler = new RetryRejectedExecutionFailureHandler)(
+ implicit f: T => ActionRequest): DataStreamSink[T] = {
process(stream, restClientFactory, failureHandler, f)
}
@@ -161,11 +172,16 @@ class ES6Sink(@(transient @param) ctx: StreamingContext, property: Properties =
process(stream, Some(restClientFactory), failureHandler, f)
}
- def sink[T](stream: JavaDataStream[T], restClientFactory: RestClientFactory, f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = {
+ def sink[T](
+ stream: JavaDataStream[T],
+ restClientFactory: RestClientFactory,
+ f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = {
process(stream, Some(restClientFactory), new RetryRejectedExecutionFailureHandler, f)
}
- def sink[T](stream: JavaDataStream[T], f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = {
+ def sink[T](
+ stream: JavaDataStream[T],
+ f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = {
process(stream, None, new RetryRejectedExecutionFailureHandler, f)
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/util/ElasticsearchUtils.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/util/ElasticsearchUtils.scala
index d84d94b81..1f91386fe 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/util/ElasticsearchUtils.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/util/ElasticsearchUtils.scala
@@ -23,15 +23,17 @@ import org.elasticsearch.common.xcontent.XContentType
object ElasticsearchUtils {
- def indexRequest(index: String, indexType: String, id: String, source: String)(implicit xContentType: XContentType = XContentType.JSON): IndexRequest = {
+ def indexRequest(index: String, indexType: String, id: String, source: String)(implicit
+ xContentType: XContentType = XContentType.JSON): IndexRequest = {
require(source != null, "indexRequest error:source can not be null...")
require(xContentType != null, "indexRequest error:xContentType can not be null...")
val indexReq = new IndexRequest(index, indexType, id)
val mapping = List("source" -> new BytesArray(source), "contentType" -> xContentType)
- mapping.foreach { x =>
- val field = indexReq.getClass.getDeclaredField(x._1)
- field.setAccessible(true)
- field.set(indexReq, x._2)
+ mapping.foreach {
+ x =>
+ val field = indexReq.getClass.getDeclaredField(x._1)
+ field.setAccessible(true)
+ field.set(indexReq, x._2)
}
indexReq
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/bean/RestClientFactoryImpl.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/bean/RestClientFactoryImpl.scala
index 3750f67e2..b56c7281e 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/bean/RestClientFactoryImpl.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/bean/RestClientFactoryImpl.scala
@@ -17,6 +17,9 @@
package org.apache.streampark.flink.connector.elasticsearch7.bean
+import org.apache.streampark.common.util.Logger
+import org.apache.streampark.flink.connector.elasticsearch7.conf.ES7Config
+
import org.apache.flink.streaming.connectors.elasticsearch7.RestClientFactory
import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials}
import org.apache.http.client.CredentialsProvider
@@ -26,9 +29,6 @@ import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.apache.http.message.BasicHeader
import org.elasticsearch.client.RestClientBuilder
-import org.apache.streampark.common.util.Logger
-import org.apache.streampark.flink.connector.elasticsearch7.conf.ES7Config
-
class RestClientFactoryImpl(val config: ES7Config) extends RestClientFactory with Logger {
override def configureRestClientBuilder(restClientBuilder: RestClientBuilder): Unit = {
// httpClientConfigCallback and requestConfigCallback........
@@ -38,17 +38,21 @@ class RestClientFactoryImpl(val config: ES7Config) extends RestClientFactory wit
// userName,password must be all set,or all not set..
require(
(userName != null && password != null) || (userName == null && password == null),
- "[StreamPark] elasticsearch auth info error,userName,password must be all set,or all not set.")
+ "[StreamPark] elasticsearch auth info error,userName,password must be all set,or all not set."
+ )
val credentialsProvider = (userName, password) match {
case (null, null) => null
case _ =>
val credentialsProvider: CredentialsProvider = new BasicCredentialsProvider()
- credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password))
+ credentialsProvider.setCredentials(
+ AuthScope.ANY,
+ new UsernamePasswordCredentials(userName, password))
credentialsProvider
}
val httpClientConfigCallback = new RestClientBuilder.HttpClientConfigCallback {
- override def customizeHttpClient(httpClientBuilder: HttpAsyncClientBuilder): HttpAsyncClientBuilder = {
+ override def customizeHttpClient(
+ httpClientBuilder: HttpAsyncClientBuilder): HttpAsyncClientBuilder = {
if (credentialsProvider != null) {
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)
logInfo("elasticsearch auth by userName,password...")
@@ -58,7 +62,8 @@ class RestClientFactoryImpl(val config: ES7Config) extends RestClientFactory wit
}
val requestConfigCallback = new RestClientBuilder.RequestConfigCallback {
- override def customizeRequestConfig(requestConfigBuilder: RequestConfig.Builder): RequestConfig.Builder = {
+ override def customizeRequestConfig(
+ requestConfigBuilder: RequestConfig.Builder): RequestConfig.Builder = {
if (credentialsProvider != null) {
requestConfigBuilder.setAuthenticationEnabled(true)
}
diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/conf/ES7Config.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/conf/ES7Config.scala
index 092116ab0..2ea777316 100644
--- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/conf/ES7Config.scala
+++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/conf/ES7Config.scala
@@ -17,10 +17,10 @@
package org.apache.streampark.flink.connector.elasticsearch7.conf
-import java.util.Properties
-
import org.apache.http.HttpHost
... 15574 lines suppressed ...