You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by kk...@apache.org on 2019/10/25 14:47:51 UTC
[flink] 01/08: [FLINK-14502] Change CustomCommandLine cluster
client methods to get a configuration
This is an automated email from the ASF dual-hosted git repository.
kkloudas pushed a commit to branch to-merge
in repository https://gitbox.apache.org/repos/asf/flink.git
commit c16b03202d56d9d2de6bc394b6b0171cb3763c56
Author: Kostas Kloudas <kk...@gmail.com>
AuthorDate: Sun Oct 13 21:09:15 2019 +0200
[FLINK-14502] Change CustomCommandLine cluster client methods to get a configuration
---
.../generated/deployment_configuration.html | 16 +
.../generated/yarn_config_configuration.html | 45 +++
docs/ops/config.md | 4 +
docs/ops/config.zh.md | 4 +
.../client/cli/AbstractCustomCommandLine.java | 9 +-
.../org/apache/flink/client/cli/CliFrontend.java | 19 +-
.../apache/flink/client/cli/CustomCommandLine.java | 36 ++-
.../org/apache/flink/client/cli/DefaultCLI.java | 12 +-
.../apache/flink/client/cli/DefaultCLITest.java | 14 +-
.../client/cli/util/DummyCustomCommandLine.java | 12 +-
.../client/program/rest/RestClusterClientTest.java | 6 +-
.../flink/configuration/DeploymentOptions.java | 35 +++
.../org/apache/flink/api/scala/FlinkShell.scala | 10 +-
.../client/gateway/local/ExecutionContext.java | 23 +-
.../flink/yarn/CliFrontendRunWithYarnTest.java | 12 +-
.../java/org/apache/flink/yarn/YARNITCase.java | 2 -
.../apache/flink/yarn/YarnConfigurationITCase.java | 2 -
.../java/org/apache/flink/yarn/YarnTestBase.java | 1 -
.../util/NonDeployingYarnClusterDescriptor.java | 3 +-
.../apache/flink/yarn/YarnClusterDescriptor.java | 42 +--
.../apache/flink/yarn/cli/FlinkYarnSessionCli.java | 347 ++++++++++++---------
.../org/apache/flink/yarn/cli/YarnConfigUtils.java | 87 ++++++
.../yarn/configuration/YarnConfigOptions.java | 48 +++
.../apache/flink/yarn/AbstractYarnClusterTest.java | 1 -
.../apache/flink/yarn/FlinkYarnSessionCliTest.java | 60 ++--
.../flink/yarn/YarnClusterDescriptorTest.java | 2 -
26 files changed, 572 insertions(+), 280 deletions(-)
diff --git a/docs/_includes/generated/deployment_configuration.html b/docs/_includes/generated/deployment_configuration.html
new file mode 100644
index 0000000..bdac757
--- /dev/null
+++ b/docs/_includes/generated/deployment_configuration.html
@@ -0,0 +1,16 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 65%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>execution.attached</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Specifies if the pipeline is submitted in attached or detached mode.</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/yarn_config_configuration.html b/docs/_includes/generated/yarn_config_configuration.html
index a28f15b..6cf3945 100644
--- a/docs/_includes/generated/yarn_config_configuration.html
+++ b/docs/_includes/generated/yarn_config_configuration.html
@@ -8,6 +8,11 @@
</thead>
<tbody>
<tr>
+ <td><h5>$internal.yarn.dynamic-properties</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>**DO NOT USE** Specify YARN dynamic properties.</td>
+ </tr>
+ <tr>
<td><h5>yarn.application-attempt-failures-validity-interval</h5></td>
<td style="word-wrap: break-word;">10000</td>
<td>Time window in milliseconds which defines the number of application attempt failures when restarting the AM. Failures which fall outside of this window are not being considered. Set this value to -1 in order to count globally. See <a href="https://hortonworks.com/blog/apache-hadoop-yarn-hdp-2-2-fault-tolerance-features-long-running-services/">here</a> for more information.</td>
@@ -23,11 +28,36 @@
<td>With this configuration option, users can specify a port, a range of ports or a list of ports for the Application Master (and JobManager) RPC port. By default we recommend using the default value (0) to let the operating system choose an appropriate port. In particular when multiple AMs are running on the same physical host, fixed port assignments prevent the AM from starting. For example when running Flink on YARN on an environment with a restrictive firewall, this optio [...]
</tr>
<tr>
+ <td><h5>yarn.application.id</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>The YARN application id of the running yarn cluster. This is the YARN cluster where the pipeline is going to be executed.</td>
+ </tr>
+ <tr>
+ <td><h5>yarn.application.name</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>A custom name for your YARN application.</td>
+ </tr>
+ <tr>
+ <td><h5>yarn.application.node-label</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>Specify YARN node label for the YARN application.</td>
+ </tr>
+ <tr>
<td><h5>yarn.application.priority</h5></td>
<td style="word-wrap: break-word;">-1</td>
<td>A non-negative integer indicating the priority for submitting a Flink YARN application. It will only take effect if YARN priority scheduling setting is enabled. Larger integer corresponds with higher priority. If priority is negative or set to '-1'(default), Flink will unset yarn priority setting and use cluster default priority. Please refer to YARN's official documentation for specific settings required to enable priority scheduling for the targeted YARN version.</td>
</tr>
<tr>
+ <td><h5>yarn.application.queue</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>The YARN queue on which to put the current pipeline.</td>
+ </tr>
+ <tr>
+ <td><h5>yarn.application.type</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>A custom type for your YARN application..</td>
+ </tr>
+ <tr>
<td><h5>yarn.appmaster.rpc.address</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>The hostname or address where the application master RPC system is listening.</td>
@@ -48,6 +78,11 @@
<td>The number of virtual cores (vcores) per YARN container. By default, the number of vcores is set to the number of slots per TaskManager, if set, or to 1, otherwise. In order for this parameter to be used your cluster must have CPU scheduling enabled. You can do this by setting the <span markdown="span">`org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler`</span>.</td>
</tr>
<tr>
+ <td><h5>yarn.flink-dist-jar</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>The location of the Flink dist jar.</td>
+ </tr>
+ <tr>
<td><h5>yarn.heartbeat.container-request-interval</h5></td>
<td style="word-wrap: break-word;">500</td>
<td>Time between heartbeats with the ResourceManager in milliseconds if Flink requests containers:<ul><li>The lower this value is, the faster Flink will get notified about container allocations since requests and allocations are transmitted via heartbeats.</li><li>The lower this value is, the more excessive containers might get allocated which will eventually be released but put pressure on Yarn.</li></ul>If you observe too many container allocations on the ResourceManager, t [...]
@@ -58,6 +93,11 @@
<td>Time between heartbeats with the ResourceManager in seconds.</td>
</tr>
<tr>
+ <td><h5>yarn.log-config-file</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>The location of the log config file, e.g. the path to your log4j.properties for log4j.</td>
+ </tr>
+ <tr>
<td><h5>yarn.maximum-failed-containers</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>Maximum number of containers the system is going to reallocate in case of a failure.</td>
@@ -73,6 +113,11 @@
<td>When a Flink job is submitted to YARN, the JobManager’s host and the number of available processing slots is written into a properties file, so that the Flink client is able to pick those details up. This configuration parameter allows changing the default location of that file (for example for environments sharing a Flink installation between users).</td>
</tr>
<tr>
+ <td><h5>yarn.ship-directories</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>A semicolon-separated list of directories to be shipped to the YARN cluster.</td>
+ </tr>
+ <tr>
<td><h5>yarn.tags</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>A comma-separated list of tags to apply to the Flink YARN application.</td>
diff --git a/docs/ops/config.md b/docs/ops/config.md
index 756153f..6a8f4ce 100644
--- a/docs/ops/config.md
+++ b/docs/ops/config.md
@@ -64,6 +64,10 @@ These parameters configure the default HDFS used by Flink. Setups that do not sp
{% include generated/core_configuration.html %}
+### Execution
+
+{% include generated/deployment_configuration.html %}
+
### JobManager
{% include generated/job_manager_configuration.html %}
diff --git a/docs/ops/config.zh.md b/docs/ops/config.zh.md
index d965e83..1274f6d 100644
--- a/docs/ops/config.zh.md
+++ b/docs/ops/config.zh.md
@@ -64,6 +64,10 @@ These parameters configure the default HDFS used by Flink. Setups that do not sp
{% include generated/core_configuration.html %}
+### Execution
+
+{% include generated/deployment_configuration.html %}
+
### JobManager
{% include generated/job_manager_configuration.html %}
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/AbstractCustomCommandLine.java b/flink-clients/src/main/java/org/apache/flink/client/cli/AbstractCustomCommandLine.java
index 0cae6e7..805d58a 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/AbstractCustomCommandLine.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/AbstractCustomCommandLine.java
@@ -69,13 +69,8 @@ public abstract class AbstractCustomCommandLine<T> implements CustomCommandLine<
baseOptions.addOption(zookeeperNamespaceOption);
}
- /**
- * Override configuration settings by specified command line options.
- *
- * @param commandLine containing the overriding values
- * @return Effective configuration with the overridden configuration settings
- */
- protected Configuration applyCommandLineOptionsToConfiguration(CommandLine commandLine) throws FlinkException {
+ @Override
+ public Configuration applyCommandLineOptionsToConfiguration(CommandLine commandLine) throws FlinkException {
final Configuration resultingConfiguration = new Configuration(configuration);
if (commandLine.hasOption(addressOption.getOpt())) {
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
index 8589227..c49374a 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
@@ -195,9 +195,9 @@ public class CliFrontend {
}
final CustomCommandLine<?> customCommandLine = getActiveCustomCommandLine(commandLine);
-
+ final Configuration executorConfig = customCommandLine.applyCommandLineOptionsToConfiguration(commandLine);
try {
- runProgram(customCommandLine, commandLine, runOptions, program);
+ runProgram(customCommandLine, executorConfig, runOptions, program);
} finally {
program.deleteExtractedLibraries();
}
@@ -205,13 +205,13 @@ public class CliFrontend {
private <T> void runProgram(
CustomCommandLine<T> customCommandLine,
- CommandLine commandLine,
+ Configuration executorConfig,
RunOptions runOptions,
PackagedProgram program) throws ProgramInvocationException, FlinkException {
- final ClusterDescriptor<T> clusterDescriptor = customCommandLine.createClusterDescriptor(commandLine);
+ final ClusterDescriptor<T> clusterDescriptor = customCommandLine.createClusterDescriptor(executorConfig);
try {
- final T clusterId = customCommandLine.getClusterId(commandLine);
+ final T clusterId = customCommandLine.getClusterId(executorConfig);
final ClusterClient<T> client;
@@ -221,7 +221,7 @@ public class CliFrontend {
final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(program, configuration, parallelism);
- final ClusterSpecification clusterSpecification = customCommandLine.getClusterSpecification(commandLine);
+ final ClusterSpecification clusterSpecification = customCommandLine.getClusterSpecification(executorConfig);
client = clusterDescriptor.deployJobCluster(
clusterSpecification,
jobGraph,
@@ -242,7 +242,7 @@ public class CliFrontend {
} else {
// also in job mode we have to deploy a session cluster because the job
// might consist of multiple parts (e.g. when using collect)
- final ClusterSpecification clusterSpecification = customCommandLine.getClusterSpecification(commandLine);
+ final ClusterSpecification clusterSpecification = customCommandLine.getClusterSpecification(executorConfig);
client = clusterDescriptor.deploySessionCluster(clusterSpecification);
// if not running in detached mode, add a shutdown hook to shut down cluster if client exits
// there's a race-condition here if cli is killed before shutdown hook is installed
@@ -912,9 +912,10 @@ public class CliFrontend {
* @throws FlinkException if something goes wrong
*/
private <T> void runClusterAction(CustomCommandLine<T> activeCommandLine, CommandLine commandLine, ClusterAction<T> clusterAction) throws FlinkException {
- final ClusterDescriptor<T> clusterDescriptor = activeCommandLine.createClusterDescriptor(commandLine);
+ final Configuration executorConfig = activeCommandLine.applyCommandLineOptionsToConfiguration(commandLine);
+ final ClusterDescriptor<T> clusterDescriptor = activeCommandLine.createClusterDescriptor(executorConfig);
- final T clusterId = activeCommandLine.getClusterId(commandLine);
+ final T clusterId = activeCommandLine.getClusterId(executorConfig);
if (clusterId == null) {
throw new FlinkException("No cluster id was specified. Please specify a cluster to which " +
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CustomCommandLine.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CustomCommandLine.java
index e939974..9458de2 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/CustomCommandLine.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CustomCommandLine.java
@@ -20,6 +20,7 @@ package org.apache.flink.client.cli;
import org.apache.flink.client.deployment.ClusterDescriptor;
import org.apache.flink.client.deployment.ClusterSpecification;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.util.FlinkException;
import org.apache.commons.cli.CommandLine;
@@ -59,37 +60,40 @@ public interface CustomCommandLine<T> {
void addGeneralOptions(Options baseOptions);
/**
- * Create a {@link ClusterDescriptor} from the given configuration, configuration directory
- * and the command line.
+ * Override configuration settings by specified command line options.
*
- * @param commandLine containing command line options relevant for the ClusterDescriptor
- * @return ClusterDescriptor
- * @throws FlinkException if the ClusterDescriptor could not be created
+ * @param commandLine containing the overriding values
+ * @return the effective configuration with the overridden configuration settings
*/
- ClusterDescriptor<T> createClusterDescriptor(CommandLine commandLine) throws FlinkException;
+ Configuration applyCommandLineOptionsToConfiguration(CommandLine commandLine) throws FlinkException;
/**
- * Returns the cluster id if a cluster id was specified on the command line, otherwise it
- * returns null.
+ * Create a {@link ClusterDescriptor} from the given configuration.
*
- * <p>A cluster id identifies a running cluster, e.g. the Yarn application id for a Flink
- * cluster running on Yarn.
+ * @param configuration containing the configuration options relevant for the {@link ClusterDescriptor}
+ * @return the corresponding {@link ClusterDescriptor}.
+ */
+ ClusterDescriptor<T> createClusterDescriptor(Configuration configuration);
+
+ /**
+ * Returns the cluster id if a cluster id is specified in the provided configuration, otherwise it returns {@code null}.
+ *
+ * <p>A cluster id identifies a running cluster, e.g. the Yarn application id for a Flink cluster running on Yarn.
*
- * @param commandLine containing command line options relevant for the cluster id retrieval
+ * @param configuration containing the configuration options relevant for the cluster id retrieval
* @return Cluster id identifying the cluster to deploy jobs to or null
*/
@Nullable
- T getClusterId(CommandLine commandLine);
+ T getClusterId(Configuration configuration);
/**
* Returns the {@link ClusterSpecification} specified by the configuration and the command
* line options. This specification can be used to deploy a new Flink cluster.
*
- * @param commandLine containing command line options relevant for the ClusterSpecification
- * @return ClusterSpecification for a new Flink cluster
- * @throws FlinkException if the ClusterSpecification could not be created
+ * @param configuration containing the configuration options relevant for the {@link ClusterSpecification}
+ * @return the corresponding {@link ClusterSpecification} for a new Flink cluster
*/
- ClusterSpecification getClusterSpecification(CommandLine commandLine) throws FlinkException;
+ ClusterSpecification getClusterSpecification(Configuration configuration);
default CommandLine parseCommandLineOptions(String[] args, boolean stopAtNonOptions) throws CliArgsException {
final Options options = new Options();
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/DefaultCLI.java b/flink-clients/src/main/java/org/apache/flink/client/cli/DefaultCLI.java
index e9ed9af..f21f755 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/DefaultCLI.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/DefaultCLI.java
@@ -22,7 +22,6 @@ import org.apache.flink.client.deployment.ClusterSpecification;
import org.apache.flink.client.deployment.StandaloneClusterDescriptor;
import org.apache.flink.client.deployment.StandaloneClusterId;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.util.FlinkException;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
@@ -55,21 +54,18 @@ public class DefaultCLI extends AbstractCustomCommandLine<StandaloneClusterId> {
}
@Override
- public StandaloneClusterDescriptor createClusterDescriptor(
- CommandLine commandLine) throws FlinkException {
- final Configuration effectiveConfiguration = applyCommandLineOptionsToConfiguration(commandLine);
-
- return new StandaloneClusterDescriptor(effectiveConfiguration);
+ public StandaloneClusterDescriptor createClusterDescriptor(Configuration configuration) {
+ return new StandaloneClusterDescriptor(configuration);
}
@Override
@Nullable
- public StandaloneClusterId getClusterId(CommandLine commandLine) {
+ public StandaloneClusterId getClusterId(Configuration configuration) {
return StandaloneClusterId.getInstance();
}
@Override
- public ClusterSpecification getClusterSpecification(CommandLine commandLine) {
+ public ClusterSpecification getClusterSpecification(Configuration configuration) {
return new ClusterSpecification.ClusterSpecificationBuilder().createClusterSpecification();
}
}
diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/DefaultCLITest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/DefaultCLITest.java
index 23fe69c..73f967d 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/cli/DefaultCLITest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/cli/DefaultCLITest.java
@@ -64,11 +64,10 @@ public class DefaultCLITest extends CliFrontendTestBase {
final String[] args = {};
CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false);
+ final Configuration executorConfig = defaultCLI.applyCommandLineOptionsToConfiguration(commandLine);
- final ClusterDescriptor<StandaloneClusterId> clusterDescriptor =
- defaultCLI.createClusterDescriptor(commandLine);
-
- final ClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(commandLine));
+ final ClusterDescriptor<StandaloneClusterId> clusterDescriptor = defaultCLI.createClusterDescriptor(executorConfig);
+ final ClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(executorConfig));
final URL webInterfaceUrl = new URL(clusterClient.getWebInterfaceURL());
@@ -97,11 +96,10 @@ public class DefaultCLITest extends CliFrontendTestBase {
final String[] args = {"-m", manualHostname + ':' + manualPort};
CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false);
+ final Configuration executorConfig = defaultCLI.applyCommandLineOptionsToConfiguration(commandLine);
- final ClusterDescriptor<StandaloneClusterId> clusterDescriptor =
- defaultCLI.createClusterDescriptor(commandLine);
-
- final ClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(commandLine));
+ final ClusterDescriptor<StandaloneClusterId> clusterDescriptor = defaultCLI.createClusterDescriptor(executorConfig);
+ final ClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(executorConfig));
final URL webInterfaceUrl = new URL(clusterClient.getWebInterfaceURL());
diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/util/DummyCustomCommandLine.java b/flink-clients/src/test/java/org/apache/flink/client/cli/util/DummyCustomCommandLine.java
index 12bea74..0bd7dc1 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/cli/util/DummyCustomCommandLine.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/cli/util/DummyCustomCommandLine.java
@@ -22,6 +22,7 @@ import org.apache.flink.client.cli.CustomCommandLine;
import org.apache.flink.client.deployment.ClusterDescriptor;
import org.apache.flink.client.deployment.ClusterSpecification;
import org.apache.flink.client.program.ClusterClient;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.util.Preconditions;
import org.apache.commons.cli.CommandLine;
@@ -60,18 +61,23 @@ public class DummyCustomCommandLine<T> implements CustomCommandLine {
}
@Override
- public ClusterDescriptor<T> createClusterDescriptor(CommandLine commandLine) {
+ public Configuration applyCommandLineOptionsToConfiguration(CommandLine commandLine) {
+ return new Configuration();
+ }
+
+ @Override
+ public ClusterDescriptor<T> createClusterDescriptor(Configuration configuration) {
return new DummyClusterDescriptor<>(clusterClient);
}
@Override
@Nullable
- public String getClusterId(CommandLine commandLine) {
+ public String getClusterId(Configuration configuration) {
return "dummy";
}
@Override
- public ClusterSpecification getClusterSpecification(CommandLine commandLine) {
+ public ClusterSpecification getClusterSpecification(Configuration configuration) {
return new ClusterSpecification.ClusterSpecificationBuilder().createClusterSpecification();
}
}
diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java
index f234e07..2ffa0b6 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java
@@ -554,10 +554,10 @@ public class RestClusterClientTest extends TestLogger {
final String[] args = {"-m", manualHostname + ':' + manualPort};
CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false);
+ final Configuration executorConfig = defaultCLI.applyCommandLineOptionsToConfiguration(commandLine);
- final StandaloneClusterDescriptor clusterDescriptor = defaultCLI.createClusterDescriptor(commandLine);
-
- final RestClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(commandLine));
+ final StandaloneClusterDescriptor clusterDescriptor = defaultCLI.createClusterDescriptor(executorConfig);
+ final RestClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(executorConfig));
URL webMonitorBaseUrl = clusterClient.getWebMonitorBaseUrl().get();
assertThat(webMonitorBaseUrl.getHost(), equalTo(manualHostname));
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/DeploymentOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/DeploymentOptions.java
new file mode 100644
index 0000000..1addcc4
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/configuration/DeploymentOptions.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.configuration;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+import static org.apache.flink.configuration.ConfigOptions.key;
+
+/**
+ * The {@link ConfigOption configuration options} relevant for all Executors.
+ */
+@PublicEvolving
+public class DeploymentOptions {
+
+ public static final ConfigOption<Boolean> ATTACHED =
+ key("execution.attached")
+ .defaultValue(false)
+ .withDescription("Specifies if the pipeline is submitted in attached or detached mode.");
+}
diff --git a/flink-scala-shell/src/main/scala/org/apache/flink/api/scala/FlinkShell.scala b/flink-scala-shell/src/main/scala/org/apache/flink/api/scala/FlinkShell.scala
index ab388c7..04f68ff 100644
--- a/flink-scala-shell/src/main/scala/org/apache/flink/api/scala/FlinkShell.scala
+++ b/flink-scala-shell/src/main/scala/org/apache/flink/api/scala/FlinkShell.scala
@@ -258,10 +258,11 @@ object FlinkShell {
val commandLine = CliFrontendParser.parse(commandLineOptions, args.toArray, true)
val customCLI = frontend.getActiveCustomCommandLine(commandLine)
+ val executorConfig = customCLI.applyCommandLineOptionsToConfiguration(commandLine);
- val clusterDescriptor = customCLI.createClusterDescriptor(commandLine)
+ val clusterDescriptor = customCLI.createClusterDescriptor(executorConfig)
- val clusterSpecification = customCLI.getClusterSpecification(commandLine)
+ val clusterSpecification = customCLI.getClusterSpecification(executorConfig)
val cluster = clusterDescriptor.deploySessionCluster(clusterSpecification)
@@ -288,12 +289,13 @@ object FlinkShell {
configuration,
CliFrontend.loadCustomCommandLines(configuration, configurationDirectory))
val customCLI = frontend.getActiveCustomCommandLine(commandLine)
+ val executorConfig = customCLI.applyCommandLineOptionsToConfiguration(commandLine);
val clusterDescriptor = customCLI
- .createClusterDescriptor(commandLine)
+ .createClusterDescriptor(executorConfig)
.asInstanceOf[ClusterDescriptor[Any]]
- val clusterId = customCLI.getClusterId(commandLine)
+ val clusterId = customCLI.getClusterId(executorConfig)
val cluster = clusterDescriptor.retrieve(clusterId)
diff --git a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java
index 73c9cce..ff54d35 100644
--- a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java
+++ b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java
@@ -110,14 +110,14 @@ public class ExecutionContext<T> {
private final Map<String, TableSink<?>> tableSinks;
private final Map<String, UserDefinedFunction> functions;
private final Configuration flinkConfig;
- private final CommandLine commandLine;
+ private final Configuration executorConfig;
private final CustomCommandLine<T> activeCommandLine;
private final RunOptions runOptions;
private final T clusterId;
private final ClusterSpecification clusterSpec;
public ExecutionContext(Environment defaultEnvironment, SessionContext sessionContext, List<URL> dependencies,
- Configuration flinkConfig, Options commandLineOptions, List<CustomCommandLine<?>> availableCommandLines) {
+ Configuration flinkConfig, Options commandLineOptions, List<CustomCommandLine<?>> availableCommandLines) throws FlinkException {
this.sessionContext = sessionContext.copy(); // create internal copy because session context is mutable
this.mergedEnv = Environment.merge(defaultEnvironment, sessionContext.getEnvironment());
this.dependencies = dependencies;
@@ -154,11 +154,12 @@ public class ExecutionContext<T> {
});
// convert deployment options into command line options that describe a cluster
- commandLine = createCommandLine(mergedEnv.getDeployment(), commandLineOptions);
+ final CommandLine commandLine = createCommandLine(mergedEnv.getDeployment(), commandLineOptions);
activeCommandLine = findActiveCommandLine(availableCommandLines, commandLine);
+ executorConfig = activeCommandLine.applyCommandLineOptionsToConfiguration(commandLine);
runOptions = createRunOptions(commandLine);
- clusterId = activeCommandLine.getClusterId(commandLine);
- clusterSpec = createClusterSpecification(activeCommandLine, commandLine);
+ clusterId = activeCommandLine.getClusterId(executorConfig);
+ clusterSpec = activeCommandLine.getClusterSpecification(executorConfig);
}
public SessionContext getSessionContext() {
@@ -181,8 +182,8 @@ public class ExecutionContext<T> {
return clusterId;
}
- public ClusterDescriptor<T> createClusterDescriptor() throws Exception {
- return activeCommandLine.createClusterDescriptor(commandLine);
+ public ClusterDescriptor<T> createClusterDescriptor() {
+ return activeCommandLine.createClusterDescriptor(executorConfig);
}
public EnvironmentInstance createEnvironmentInstance() {
@@ -243,14 +244,6 @@ public class ExecutionContext<T> {
}
}
- private static ClusterSpecification createClusterSpecification(CustomCommandLine<?> activeCommandLine, CommandLine commandLine) {
- try {
- return activeCommandLine.getClusterSpecification(commandLine);
- } catch (FlinkException e) {
- throw new SqlExecutionException("Could not create cluster specification for the given deployment.", e);
- }
- }
-
private Catalog createCatalog(String name, Map<String, String> catalogProperties, ClassLoader classLoader) {
final CatalogFactory factory =
TableFactoryService.find(CatalogFactory.class, catalogProperties, classLoader);
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/CliFrontendRunWithYarnTest.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/CliFrontendRunWithYarnTest.java
index 28a8d5a..e16abc0 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/CliFrontendRunWithYarnTest.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/CliFrontendRunWithYarnTest.java
@@ -23,12 +23,10 @@ import org.apache.flink.client.cli.CliFrontendTestUtils;
import org.apache.flink.client.program.ClusterClient;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.JobManagerOptions;
-import org.apache.flink.util.FlinkException;
import org.apache.flink.yarn.cli.FlinkYarnSessionCli;
import org.apache.flink.yarn.util.FakeClusterClient;
import org.apache.flink.yarn.util.NonDeployingYarnClusterDescriptor;
-import org.apache.commons.cli.CommandLine;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.AfterClass;
@@ -90,7 +88,6 @@ public class CliFrontendRunWithYarnTest extends CliFrontendTestBase {
private static class TestingFlinkYarnSessionCli extends FlinkYarnSessionCli {
private final ClusterClient<ApplicationId> clusterClient;
- private final String configurationDirectory;
private TestingFlinkYarnSessionCli(
Configuration configuration,
@@ -99,18 +96,15 @@ public class CliFrontendRunWithYarnTest extends CliFrontendTestBase {
String longPrefix) throws Exception {
super(configuration, configurationDirectory, shortPrefix, longPrefix);
- this.clusterClient = new FakeClusterClient();
- this.configurationDirectory = configurationDirectory;
+ this.clusterClient = new FakeClusterClient(configuration);
}
@Override
- public YarnClusterDescriptor createClusterDescriptor(CommandLine commandLine)
- throws FlinkException {
- YarnClusterDescriptor parent = super.createClusterDescriptor(commandLine);
+ public YarnClusterDescriptor createClusterDescriptor(Configuration configuration) {
+ YarnClusterDescriptor parent = super.createClusterDescriptor(configuration);
return new NonDeployingYarnClusterDescriptor(
parent.getFlinkConfiguration(),
(YarnConfiguration) parent.getYarnClient().getConfig(),
- configurationDirectory,
parent.getYarnClient(),
clusterClient);
}
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
index e894f65..6cf6fb6 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
@@ -22,7 +22,6 @@ import org.apache.flink.api.common.time.Deadline;
import org.apache.flink.client.deployment.ClusterSpecification;
import org.apache.flink.client.program.ClusterClient;
import org.apache.flink.configuration.AkkaOptions;
-import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobmaster.JobResult;
@@ -72,7 +71,6 @@ public class YARNITCase extends YarnTestBase {
try (final YarnClusterDescriptor yarnClusterDescriptor = new YarnClusterDescriptor(
configuration,
getYarnConfiguration(),
- System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR),
yarnClient,
true)) {
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java
index 7459bac..2aa476a 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java
@@ -19,7 +19,6 @@
package org.apache.flink.yarn;
import org.apache.flink.api.common.time.Time;
-import org.apache.flink.client.cli.CliFrontend;
import org.apache.flink.client.deployment.ClusterSpecification;
import org.apache.flink.client.program.ClusterClient;
import org.apache.flink.client.program.PackagedProgram;
@@ -98,7 +97,6 @@ public class YarnConfigurationITCase extends YarnTestBase {
final YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(
configuration,
yarnConfiguration,
- CliFrontend.getConfigurationDirectoryFromEnv(),
yarnClient,
true);
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
index 6481420..80f6efc 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
@@ -307,7 +307,6 @@ public abstract class YarnTestBase extends TestLogger {
final YarnClusterDescriptor yarnClusterDescriptor = new YarnClusterDescriptor(
flinkConfiguration,
YARN_CONFIGURATION,
- CliFrontend.getConfigurationDirectoryFromEnv(),
yarnClient,
true);
yarnClusterDescriptor.setLocalJarPath(new Path(flinkUberjar.toURI()));
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/NonDeployingYarnClusterDescriptor.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/NonDeployingYarnClusterDescriptor.java
index 2671936..48e05f8 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/NonDeployingYarnClusterDescriptor.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/NonDeployingYarnClusterDescriptor.java
@@ -40,10 +40,9 @@ public class NonDeployingYarnClusterDescriptor extends YarnClusterDescriptor {
public NonDeployingYarnClusterDescriptor(
Configuration flinkConfiguration,
YarnConfiguration yarnConfiguration,
- String configurationDirectory,
YarnClient yarnClient,
ClusterClient<ApplicationId> clusterClient) {
- super(flinkConfiguration, yarnConfiguration, configurationDirectory, yarnClient, true);
+ super(flinkConfiguration, yarnConfiguration, yarnClient, true);
this.clusterClient = Preconditions.checkNotNull(clusterClient);
}
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java
index 85d420f..68eea73 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java
@@ -46,6 +46,7 @@ import org.apache.flink.runtime.taskexecutor.TaskManagerServices;
import org.apache.flink.util.FlinkException;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.ShutdownHookUtil;
+import org.apache.flink.yarn.cli.YarnConfigUtils;
import org.apache.flink.yarn.configuration.YarnConfigOptions;
import org.apache.flink.yarn.entrypoint.YarnJobClusterEntrypoint;
import org.apache.flink.yarn.entrypoint.YarnSessionClusterEntrypoint;
@@ -129,8 +130,6 @@ public class YarnClusterDescriptor implements ClusterDescriptor<ApplicationId> {
private String yarnQueue;
- private String configurationDirectory;
-
private Path flinkJarPath;
private String dynamicPropertiesEncoded;
@@ -152,7 +151,6 @@ public class YarnClusterDescriptor implements ClusterDescriptor<ApplicationId> {
public YarnClusterDescriptor(
Configuration flinkConfiguration,
YarnConfiguration yarnConfiguration,
- String configurationDirectory,
YarnClient yarnClient,
boolean sharedYarnClient) {
@@ -162,8 +160,6 @@ public class YarnClusterDescriptor implements ClusterDescriptor<ApplicationId> {
this.flinkConfiguration = Preconditions.checkNotNull(flinkConfiguration);
this.userJarInclusion = getUserJarInclusionMode(flinkConfiguration);
-
- this.configurationDirectory = Preconditions.checkNotNull(configurationDirectory);
}
@VisibleForTesting
@@ -235,9 +231,6 @@ public class YarnClusterDescriptor implements ClusterDescriptor<ApplicationId> {
if (this.flinkJarPath == null) {
throw new YarnDeploymentException("The Flink jar path is null");
}
- if (this.configurationDirectory == null) {
- throw new YarnDeploymentException("Configuration directory not set");
- }
if (this.flinkConfiguration == null) {
throw new YarnDeploymentException("Flink configuration object has not been set");
}
@@ -674,6 +667,15 @@ public class YarnClusterDescriptor implements ClusterDescriptor<ApplicationId> {
}
}
+ private boolean containsFileWithEnding(final Set<File> files, final String suffix) {
+ for (File file: files) {
+ if (file.getPath().endsWith(suffix)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private ApplicationReport startAppMaster(
Configuration configuration,
String applicationName,
@@ -709,22 +711,10 @@ public class YarnClusterDescriptor implements ClusterDescriptor<ApplicationId> {
systemShipFiles.add(file.getAbsoluteFile());
}
- //check if there is a logback or log4j file
- File logbackFile = new File(configurationDirectory + File.separator + CONFIG_FILE_LOGBACK_NAME);
- final boolean hasLogback = logbackFile.exists();
- if (hasLogback) {
- systemShipFiles.add(logbackFile);
- }
-
- File log4jFile = new File(configurationDirectory + File.separator + CONFIG_FILE_LOG4J_NAME);
- final boolean hasLog4j = log4jFile.exists();
- if (hasLog4j) {
- systemShipFiles.add(log4jFile);
- if (hasLogback) {
- // this means there is already a logback configuration file --> fail
- LOG.warn("The configuration directory ('" + configurationDirectory + "') contains both LOG4J and " +
- "Logback configuration files. Please delete or rename one of them.");
- }
+ final List<File> logConfigFiles = YarnConfigUtils
+ .decodeListFromConfig(configuration, YarnConfigOptions.APPLICATION_LOG_CONFIG_FILES, File::new);
+ if (logConfigFiles != null) {
+ systemShipFiles.addAll(logConfigFiles);
}
addEnvironmentFoldersToShipFiles(systemShipFiles);
@@ -944,8 +934,8 @@ public class YarnClusterDescriptor implements ClusterDescriptor<ApplicationId> {
final ContainerLaunchContext amContainer = setupApplicationMasterContainer(
yarnClusterEntrypoint,
- hasLogback,
- hasLog4j,
+ containsFileWithEnding(systemShipFiles, CONFIG_FILE_LOGBACK_NAME),
+ containsFileWithEnding(systemShipFiles, CONFIG_FILE_LOG4J_NAME),
hasKrb5,
clusterSpecification.getMasterMemoryMB());
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
index 8d6916a..61e3d58 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
@@ -26,6 +26,7 @@ import org.apache.flink.client.program.ClusterClient;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.ConfigurationUtils;
import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.DeploymentOptions;
import org.apache.flink.configuration.GlobalConfiguration;
import org.apache.flink.configuration.JobManagerOptions;
import org.apache.flink.configuration.MemorySize;
@@ -37,7 +38,6 @@ import org.apache.flink.runtime.security.SecurityUtils;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.ExecutorUtils;
import org.apache.flink.util.FlinkException;
-import org.apache.flink.util.Preconditions;
import org.apache.flink.util.ShutdownHookUtil;
import org.apache.flink.yarn.YarnClusterDescriptor;
import org.apache.flink.yarn.configuration.YarnConfigOptions;
@@ -52,7 +52,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.slf4j.Logger;
@@ -72,12 +71,13 @@ import java.io.UnsupportedEncodingException;
import java.lang.reflect.UndeclaredThrowableException;
import java.net.URLDecoder;
import java.nio.charset.Charset;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -87,6 +87,7 @@ import static org.apache.flink.client.cli.CliFrontendParser.DETACHED_OPTION;
import static org.apache.flink.client.cli.CliFrontendParser.SHUTDOWN_IF_ATTACHED_OPTION;
import static org.apache.flink.client.cli.CliFrontendParser.YARN_DETACHED_OPTION;
import static org.apache.flink.configuration.HighAvailabilityOptions.HA_CLUSTER_ID;
+import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* Class handling the command line interface to the YARN session.
@@ -158,8 +159,6 @@ public class FlinkYarnSessionCli extends AbstractCustomCommandLine<ApplicationId
private final String yarnPropertiesFileLocation;
- private final YarnConfiguration yarnConfiguration;
-
public FlinkYarnSessionCli(
Configuration configuration,
String configurationDirectory,
@@ -175,7 +174,7 @@ public class FlinkYarnSessionCli extends AbstractCustomCommandLine<ApplicationId
String longPrefix,
boolean acceptInteractiveInput) throws FlinkException {
super(configuration);
- this.configurationDirectory = Preconditions.checkNotNull(configurationDirectory);
+ this.configurationDirectory = checkNotNull(configurationDirectory);
this.acceptInteractiveInput = acceptInteractiveInput;
// Create the command line options
@@ -256,126 +255,118 @@ public class FlinkYarnSessionCli extends AbstractCustomCommandLine<ApplicationId
} else {
yarnApplicationIdFromYarnProperties = null;
}
-
- this.yarnConfiguration = new YarnConfiguration();
}
- private YarnClusterDescriptor createDescriptor(
- Configuration configuration,
- YarnConfiguration yarnConfiguration,
- String configurationDirectory,
- CommandLine cmd) {
-
- YarnClusterDescriptor yarnClusterDescriptor = getClusterDescriptor(
- configuration,
- yarnConfiguration,
- configurationDirectory);
-
- // Jar Path
- final Path localJarPath;
- if (cmd.hasOption(flinkJar.getOpt())) {
- String userPath = cmd.getOptionValue(flinkJar.getOpt());
- if (!userPath.startsWith("file://")) {
- userPath = "file://" + userPath;
- }
- localJarPath = new Path(userPath);
- } else {
- LOG.info("No path for the flink jar passed. Using the location of "
- + yarnClusterDescriptor.getClass() + " to locate the jar");
- String encodedJarPath =
- yarnClusterDescriptor.getClass().getProtectionDomain().getCodeSource().getLocation().getPath();
-
- final String decodedPath;
- try {
- // we have to decode the url encoded parts of the path
- decodedPath = URLDecoder.decode(encodedJarPath, Charset.defaultCharset().name());
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException("Couldn't decode the encoded Flink dist jar path: " + encodedJarPath +
- " Please supply a path manually via the -" + flinkJar.getOpt() + " option.");
- }
-
- // check whether it's actually a jar file --> when testing we execute this class without a flink-dist jar
- if (decodedPath.endsWith(".jar")) {
- localJarPath = new Path(new File(decodedPath).toURI());
- } else {
- localJarPath = null;
- }
- }
+ private YarnClusterDescriptor createDescriptor(Configuration configuration) {
+ YarnClusterDescriptor yarnClusterDescriptor = new YarnClusterDescriptor(configuration);
+ final Path localJarPath = getLocalFlinkDistPath(configuration, yarnClusterDescriptor);
if (localJarPath != null) {
yarnClusterDescriptor.setLocalJarPath(localJarPath);
}
- List<File> shipFiles = new ArrayList<>();
- // path to directories to ship
- if (cmd.hasOption(shipPath.getOpt())) {
- String[] shipPaths = cmd.getOptionValues(this.shipPath.getOpt());
- for (String shipPath : shipPaths) {
- File shipDir = new File(shipPath);
- if (shipDir.isDirectory()) {
- shipFiles.add(shipDir);
- } else {
- LOG.warn("Ship directory {} is not a directory. Ignoring it.", shipDir.getAbsolutePath());
- }
- }
- }
-
+ final List<File> shipFiles = decodeDirsToShipToCluster(configuration);
yarnClusterDescriptor.addShipFiles(shipFiles);
- // queue
- if (cmd.hasOption(queue.getOpt())) {
- yarnClusterDescriptor.setQueue(cmd.getOptionValue(queue.getOpt()));
+ final String queueName = configuration.getString(YarnConfigOptions.APPLICATION_QUEUE);
+ if (queueName != null) {
+ yarnClusterDescriptor.setQueue(queueName);
}
- final Properties properties = cmd.getOptionProperties(dynamicproperties.getOpt());
+ final String dynamicPropertiesEncoded = configuration.getString(YarnConfigOptions.DYNAMIC_PROPERTIES);
+ yarnClusterDescriptor.setDynamicPropertiesEncoded(dynamicPropertiesEncoded);
- for (String key : properties.stringPropertyNames()) {
- LOG.info("Dynamic Property set: {}={}", key, GlobalConfiguration.isSensitive(key) ? GlobalConfiguration.HIDDEN_CONTENT : properties.getProperty(key));
+ final boolean detached = !configuration.getBoolean(DeploymentOptions.ATTACHED);
+ yarnClusterDescriptor.setDetachedMode(detached);
+
+ final String appName = configuration.getString(YarnConfigOptions.APPLICATION_NAME);
+ if (appName != null) {
+ yarnClusterDescriptor.setName(appName);
}
- String[] dynamicProperties = properties.stringPropertyNames().stream()
- .flatMap(
- (String key) -> {
- final String value = properties.getProperty(key);
+ final String appType = configuration.getString(YarnConfigOptions.APPLICATION_TYPE);
+ if (appType != null) {
+ yarnClusterDescriptor.setApplicationType(appType);
+ }
- if (value != null) {
- return Stream.of(key + dynamicproperties.getValueSeparator() + value);
- } else {
- return Stream.empty();
- }
- })
- .toArray(String[]::new);
+ final String zkNamespace = configuration.getString(HA_CLUSTER_ID);
+ if (zkNamespace != null) {
+ yarnClusterDescriptor.setZookeeperNamespace(zkNamespace);
+ }
- String dynamicPropertiesEncoded = StringUtils.join(dynamicProperties, YARN_DYNAMIC_PROPERTIES_SEPARATOR);
+ final String nodeLabel = configuration.getString(YarnConfigOptions.NODE_LABEL);
+ if (nodeLabel != null) {
+ yarnClusterDescriptor.setNodeLabel(nodeLabel);
+ }
- yarnClusterDescriptor.setDynamicPropertiesEncoded(dynamicPropertiesEncoded);
+ return yarnClusterDescriptor;
+ }
- if (cmd.hasOption(YARN_DETACHED_OPTION.getOpt()) || cmd.hasOption(DETACHED_OPTION.getOpt())) {
- yarnClusterDescriptor.setDetachedMode(true);
+ private Path getLocalFlinkDistPath(final Configuration configuration, final YarnClusterDescriptor yarnClusterDescriptor) {
+ final String localJarPath = configuration.getString(YarnConfigOptions.FLINK_DIST_JAR);
+ if (localJarPath != null) {
+ return new Path(localJarPath);
}
- if (cmd.hasOption(name.getOpt())) {
- yarnClusterDescriptor.setName(cmd.getOptionValue(name.getOpt()));
- }
+ LOG.info("No path for the flink jar passed. Using the location of " + yarnClusterDescriptor.getClass() + " to locate the jar");
+
+ // check whether it's actually a jar file --> when testing we execute this class without a flink-dist jar
+ final String decodedPath = getDecodedJarPath(yarnClusterDescriptor);
+ return decodedPath.endsWith(".jar")
+ ? new Path(new File(decodedPath).toURI())
+ : null;
+ }
- if (cmd.hasOption(applicationType.getOpt())) {
- yarnClusterDescriptor.setApplicationType(cmd.getOptionValue(applicationType.getOpt()));
+ private String getDecodedJarPath(final YarnClusterDescriptor yarnClusterDescriptor) {
+ final String encodedJarPath = yarnClusterDescriptor
+ .getClass().getProtectionDomain().getCodeSource().getLocation().getPath();
+ try {
+ return URLDecoder.decode(encodedJarPath, Charset.defaultCharset().name());
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException("Couldn't decode the encoded Flink dist jar path: " + encodedJarPath +
+ " Please supply a path manually via the -" + flinkJar.getOpt() + " option.");
}
+ }
- if (cmd.hasOption(zookeeperNamespace.getOpt())) {
- String zookeeperNamespaceValue = cmd.getOptionValue(this.zookeeperNamespace.getOpt());
- yarnClusterDescriptor.setZookeeperNamespace(zookeeperNamespaceValue);
+ private Path getLocalFlinkDistPathFromCmd(final CommandLine cmd) {
+ final String flinkJarOptionName = flinkJar.getOpt();
+ if (!cmd.hasOption(flinkJarOptionName)) {
+ return null;
}
- if (cmd.hasOption(nodeLabel.getOpt())) {
- String nodeLabelValue = cmd.getOptionValue(this.nodeLabel.getOpt());
- yarnClusterDescriptor.setNodeLabel(nodeLabelValue);
+ String userPath = cmd.getOptionValue(flinkJarOptionName);
+ if (!userPath.startsWith("file://")) {
+ userPath = "file://" + userPath;
}
+ return new Path(userPath);
+ }
- return yarnClusterDescriptor;
+ private List<File> decodeDirsToShipToCluster(final Configuration configuration) {
+ checkNotNull(configuration);
+ return YarnConfigUtils.decodeListFromConfig(configuration, YarnConfigOptions.SHIP_DIRECTORIES, File::new);
+ }
+
+ private void encodeDirsToShipToCluster(final Configuration configuration, final CommandLine cmd) {
+ checkNotNull(cmd);
+ checkNotNull(configuration);
+
+ if (cmd.hasOption(shipPath.getOpt())) {
+ YarnConfigUtils.encodeListToConfig(
+ configuration,
+ YarnConfigOptions.SHIP_DIRECTORIES,
+ cmd.getOptionValues(this.shipPath.getOpt()),
+ (String path) -> {
+ final File shipDir = new File(path);
+ if (shipDir.isDirectory()) {
+ return path;
+ }
+ LOG.warn("Ship directory {} is not a directory. Ignoring it.", shipDir.getAbsolutePath());
+ return null;
+ });
+ }
}
- private ClusterSpecification createClusterSpecification(Configuration configuration, CommandLine cmd) {
+ private ClusterSpecification createClusterSpecification(Configuration configuration) {
// JobManager Memory
final int jobManagerMemoryMB = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes();
@@ -433,47 +424,30 @@ public class FlinkYarnSessionCli extends AbstractCustomCommandLine<ApplicationId
}
@Override
- public YarnClusterDescriptor createClusterDescriptor(CommandLine commandLine) throws FlinkException {
- final Configuration effectiveConfiguration = applyCommandLineOptionsToConfiguration(commandLine);
-
- return createDescriptor(
- effectiveConfiguration,
- yarnConfiguration,
- configurationDirectory,
- commandLine);
+ public YarnClusterDescriptor createClusterDescriptor(Configuration configuration) {
+ return createDescriptor(configuration);
}
@Override
@Nullable
- public ApplicationId getClusterId(CommandLine commandLine) {
- if (commandLine.hasOption(applicationId.getOpt())) {
- return ConverterUtils.toApplicationId(commandLine.getOptionValue(applicationId.getOpt()));
- } else if (isYarnPropertiesFileMode(commandLine)) {
- return yarnApplicationIdFromYarnProperties;
- } else {
- return null;
- }
+ public ApplicationId getClusterId(Configuration configuration) {
+ final String clusterId = configuration.getString(YarnConfigOptions.APPLICATION_ID);
+ return clusterId != null ? ConverterUtils.toApplicationId(clusterId) : null;
}
@Override
- public ClusterSpecification getClusterSpecification(CommandLine commandLine) throws FlinkException {
- final Configuration effectiveConfiguration = applyCommandLineOptionsToConfiguration(commandLine);
-
- return createClusterSpecification(effectiveConfiguration, commandLine);
+ public ClusterSpecification getClusterSpecification(Configuration configuration) {
+ return createClusterSpecification(configuration);
}
@Override
- protected Configuration applyCommandLineOptionsToConfiguration(CommandLine commandLine) throws FlinkException {
+ public Configuration applyCommandLineOptionsToConfiguration(CommandLine commandLine) throws FlinkException {
// we ignore the addressOption because it can only contain "yarn-cluster"
final Configuration effectiveConfiguration = new Configuration(configuration);
- if (commandLine.hasOption(zookeeperNamespaceOption.getOpt())) {
- String zkNamespace = commandLine.getOptionValue(zookeeperNamespaceOption.getOpt());
- effectiveConfiguration.setString(HA_CLUSTER_ID, zkNamespace);
- }
-
- final ApplicationId applicationId = getClusterId(commandLine);
+ applyDescriptorOptionToConfig(commandLine, effectiveConfiguration);
+ final ApplicationId applicationId = getApplicationId(commandLine);
if (applicationId != null) {
final String zooKeeperNamespace;
if (commandLine.hasOption(zookeeperNamespace.getOpt())){
@@ -483,6 +457,7 @@ public class FlinkYarnSessionCli extends AbstractCustomCommandLine<ApplicationId
}
effectiveConfiguration.setString(HA_CLUSTER_ID, zooKeeperNamespace);
+ effectiveConfiguration.setString(YarnConfigOptions.APPLICATION_ID, ConverterUtils.toString(applicationId));
}
if (commandLine.hasOption(jmMemory.getOpt())) {
@@ -512,6 +487,89 @@ public class FlinkYarnSessionCli extends AbstractCustomCommandLine<ApplicationId
}
}
+ private ApplicationId getApplicationId(CommandLine commandLine) {
+ if (commandLine.hasOption(applicationId.getOpt())) {
+ return ConverterUtils.toApplicationId(commandLine.getOptionValue(applicationId.getOpt()));
+ } else if (isYarnPropertiesFileMode(commandLine)) {
+ return yarnApplicationIdFromYarnProperties;
+ }
+ return null;
+ }
+
+ private void applyDescriptorOptionToConfig(final CommandLine commandLine, final Configuration configuration) {
+ checkNotNull(commandLine);
+ checkNotNull(configuration);
+
+ final Path localJarPath = getLocalFlinkDistPathFromCmd(commandLine);
+ if (localJarPath != null) {
+ configuration.setString(YarnConfigOptions.FLINK_DIST_JAR, localJarPath.toString());
+ }
+
+ encodeDirsToShipToCluster(configuration, commandLine);
+
+ if (commandLine.hasOption(queue.getOpt())) {
+ final String queueName = commandLine.getOptionValue(queue.getOpt());
+ configuration.setString(YarnConfigOptions.APPLICATION_QUEUE, queueName);
+ }
+
+ final String dynamicPropertiesEncoded = encodeDynamicProperties(commandLine);
+ configuration.setString(YarnConfigOptions.DYNAMIC_PROPERTIES, dynamicPropertiesEncoded);
+
+ final boolean detached = commandLine.hasOption(YARN_DETACHED_OPTION.getOpt()) || commandLine.hasOption(DETACHED_OPTION.getOpt());
+ configuration.setBoolean(DeploymentOptions.ATTACHED, !detached);
+
+ if (commandLine.hasOption(name.getOpt())) {
+ final String appName = commandLine.getOptionValue(name.getOpt());
+ configuration.setString(YarnConfigOptions.APPLICATION_NAME, appName);
+ }
+
+ if (commandLine.hasOption(applicationType.getOpt())) {
+ final String appType = commandLine.getOptionValue(applicationType.getOpt());
+ configuration.setString(YarnConfigOptions.APPLICATION_TYPE, appType);
+ }
+
+ if (commandLine.hasOption(zookeeperNamespace.getOpt())) {
+ String zookeeperNamespaceValue = commandLine.getOptionValue(zookeeperNamespace.getOpt());
+ configuration.setString(HA_CLUSTER_ID, zookeeperNamespaceValue);
+ } else if (commandLine.hasOption(zookeeperNamespaceOption.getOpt())) {
+ String zookeeperNamespaceValue = commandLine.getOptionValue(zookeeperNamespaceOption.getOpt());
+ configuration.setString(HA_CLUSTER_ID, zookeeperNamespaceValue);
+ }
+
+ if (commandLine.hasOption(nodeLabel.getOpt())) {
+ final String nodeLabelValue = commandLine.getOptionValue(this.nodeLabel.getOpt());
+ configuration.setString(YarnConfigOptions.NODE_LABEL, nodeLabelValue);
+ }
+
+ discoverAndEncodeLogConfigFiles(configuration);
+ }
+
+ private void discoverAndEncodeLogConfigFiles(final Configuration configuration) {
+ final Set<File> logFiles = discoverLogConfigFiles();
+ YarnConfigUtils.encodeListToConfig(configuration, YarnConfigOptions.APPLICATION_LOG_CONFIG_FILES, logFiles, File::getPath);
+ }
+
+ private Set<File> discoverLogConfigFiles() {
+ final Set<File> logConfigFiles = new HashSet<>();
+
+ File logbackFile = new File(configurationDirectory + File.separator + CONFIG_FILE_LOGBACK_NAME);
+ final boolean hasLogback = logbackFile.exists();
+ if (hasLogback) {
+ logConfigFiles.add(logbackFile);
+ }
+
+ File log4jFile = new File(configurationDirectory + File.separator + CONFIG_FILE_LOG4J_NAME);
+ final boolean hasLog4j = log4jFile.exists();
+ if (hasLog4j) {
+ logConfigFiles.add(log4jFile);
+ if (hasLogback) {
+ LOG.warn("The configuration directory ('" + configurationDirectory + "') contains both LOG4J and " +
+ "Logback configuration files. Please delete or rename one of them.");
+ }
+ }
+ return logConfigFiles;
+ }
+
private boolean isYarnPropertiesFileMode(CommandLine commandLine) {
boolean canApplyYarnProperties = !commandLine.hasOption(addressOption.getOpt());
@@ -573,7 +631,8 @@ public class FlinkYarnSessionCli extends AbstractCustomCommandLine<ApplicationId
return 0;
}
- final YarnClusterDescriptor yarnClusterDescriptor = createClusterDescriptor(cmd);
+ final Configuration configuration = applyCommandLineOptionsToConfiguration(cmd);
+ final YarnClusterDescriptor yarnClusterDescriptor = createClusterDescriptor(configuration);
try {
// Query cluster for metrics
@@ -590,7 +649,7 @@ public class FlinkYarnSessionCli extends AbstractCustomCommandLine<ApplicationId
clusterClient = yarnClusterDescriptor.retrieve(yarnApplicationId);
} else {
- final ClusterSpecification clusterSpecification = getClusterSpecification(cmd);
+ final ClusterSpecification clusterSpecification = getClusterSpecification(configuration);
clusterClient = yarnClusterDescriptor.deploySessionCluster(clusterSpecification);
@@ -772,6 +831,26 @@ public class FlinkYarnSessionCli extends AbstractCustomCommandLine<ApplicationId
System.out.println(message);
}
+ private String encodeDynamicProperties(final CommandLine cmd) {
+ final Properties properties = cmd.getOptionProperties(dynamicproperties.getOpt());
+ final String[] dynamicProperties = properties.stringPropertyNames().stream()
+ .flatMap(
+ (String key) -> {
+ final String value = properties.getProperty(key);
+
+ LOG.info("Dynamic Property set: {}={}", key, GlobalConfiguration.isSensitive(key) ? GlobalConfiguration.HIDDEN_CONTENT : value);
+
+ if (value != null) {
+ return Stream.of(key + dynamicproperties.getValueSeparator() + value);
+ } else {
+ return Stream.empty();
+ }
+ })
+ .toArray(String[]::new);
+
+ return StringUtils.join(dynamicProperties, YARN_DYNAMIC_PROPERTIES_SEPARATOR);
+ }
+
public static Map<String, String> getDynamicProperties(String dynamicPropertiesEncoded) {
if (dynamicPropertiesEncoded != null && dynamicPropertiesEncoded.length() > 0) {
Map<String, String> properties = new HashMap<>();
@@ -958,20 +1037,4 @@ public class FlinkYarnSessionCli extends AbstractCustomCommandLine<ApplicationId
return new File(propertiesFileLocation, YARN_PROPERTIES_FILE + currentUser);
}
-
- private YarnClusterDescriptor getClusterDescriptor(
- Configuration configuration,
- YarnConfiguration yarnConfiguration,
- String configurationDirectory) {
- final YarnClient yarnClient = YarnClient.createYarnClient();
- yarnClient.init(yarnConfiguration);
- yarnClient.start();
-
- return new YarnClusterDescriptor(
- configuration,
- yarnConfiguration,
- configurationDirectory,
- yarnClient,
- false);
- }
}
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/YarnConfigUtils.java b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/YarnConfigUtils.java
new file mode 100644
index 0000000..edb0209
--- /dev/null
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/YarnConfigUtils.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn.cli;
+
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Utilities for parsing {@link org.apache.flink.configuration.ConfigOption configuration options}.
+ */
+public class YarnConfigUtils {
+ // TODO: 16.10.19 test this and test LOG FILES discovery.
+ private static final String COLLECTION_DELIMITER = ";";
+
+ public static <T> void encodeListToConfig(
+ final Configuration configuration,
+ final ConfigOption<String> key,
+ final Collection<T> value,
+ final Function<T, String> mapper) {
+ encodeListToConfig(configuration, key, value.stream(), mapper);
+ }
+
+ public static <T> void encodeListToConfig(
+ final Configuration configuration,
+ final ConfigOption<String> key,
+ final T[] value,
+ final Function<T, String> mapper) {
+ encodeListToConfig(configuration, key, Arrays.stream(value), mapper);
+ }
+
+ private static <T> void encodeListToConfig(
+ final Configuration configuration,
+ final ConfigOption<String> key,
+ final Stream<T> values,
+ final Function<T, String> mapper) {
+
+ checkNotNull(values);
+ checkNotNull(key);
+ checkNotNull(configuration);
+
+ final String encodedString = values.map(mapper).filter(Objects::nonNull).collect(Collectors.joining(COLLECTION_DELIMITER));
+ if (encodedString != null && !encodedString.isEmpty()) {
+ configuration.setString(key, encodedString);
+ }
+ }
+
+ public static <R> List<R> decodeListFromConfig(
+ final Configuration configuration,
+ final ConfigOption<String> key,
+ final Function<String, R> mapper) {
+
+ checkNotNull(configuration);
+ checkNotNull(key);
+
+ final String encodedString = configuration.getString(key);
+ return encodedString != null
+ ? Arrays.stream(encodedString.split(COLLECTION_DELIMITER)).map(mapper).collect(Collectors.toList())
+ : Collections.emptyList();
+ }
+}
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java b/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
index 035bb04..d8611cc 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
@@ -204,6 +204,54 @@ public class YarnConfigOptions {
.defaultValue("")
.withDescription("A comma-separated list of tags to apply to the Flink YARN application.");
+ // ----------------------- YARN CLI OPTIONS ------------------------------------
+
+ public static final ConfigOption<String> APPLICATION_LOG_CONFIG_FILES =
+ key("yarn.log-config-file")
+ .noDefaultValue()
+ .withDescription("The location of the log config file, e.g. the path to your log4j.properties for log4j.");
+
+ public static final ConfigOption<String> DYNAMIC_PROPERTIES =
+ key("$internal.yarn.dynamic-properties")
+ .noDefaultValue()
+ .withDescription("**DO NOT USE** Specify YARN dynamic properties.");
+
+ public static final ConfigOption<String> SHIP_DIRECTORIES =
+ key("yarn.ship-directories")
+ .noDefaultValue()
+ .withDescription("A semicolon-separated list of directories to be shipped to the YARN cluster.");
+
+ public static final ConfigOption<String> FLINK_DIST_JAR =
+ key("yarn.flink-dist-jar")
+ .noDefaultValue()
+ .withDescription("The location of the Flink dist jar.");
+
+ public static final ConfigOption<String> APPLICATION_ID =
+ key("yarn.application.id")
+ .noDefaultValue()
+ .withDescription("The YARN application id of the running yarn cluster." +
+ " This is the YARN cluster where the pipeline is going to be executed.");
+
+ public static final ConfigOption<String> APPLICATION_QUEUE =
+ key("yarn.application.queue")
+ .noDefaultValue()
+ .withDescription("The YARN queue on which to put the current pipeline.");
+
+ public static final ConfigOption<String> APPLICATION_NAME =
+ key("yarn.application.name")
+ .noDefaultValue()
+ .withDescription("A custom name for your YARN application.");
+
+ public static final ConfigOption<String> APPLICATION_TYPE =
+ key("yarn.application.type")
+ .noDefaultValue()
+ .withDescription("A custom type for your YARN application..");
+
+ public static final ConfigOption<String> NODE_LABEL =
+ key("yarn.application.node-label")
+ .noDefaultValue()
+ .withDescription("Specify YARN node label for the YARN application.");
+
// ------------------------------------------------------------------------
/** This class is not meant to be instantiated. */
diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
index 9009a53..1ba2008 100644
--- a/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
+++ b/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
@@ -67,7 +67,6 @@ public class AbstractYarnClusterTest extends TestLogger {
final YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(
new Configuration(),
yarnConfiguration,
- temporaryFolder.newFolder().getAbsolutePath(),
yarnClient,
false);
diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java
index 2338875..ae566b4 100644
--- a/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java
+++ b/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java
@@ -85,7 +85,8 @@ public class FlinkYarnSessionCliTest extends TestLogger {
CommandLine cmd = parser.parse(options, new String[]{"run", "-j", "fake.jar",
"-D", "akka.ask.timeout=5 min", "-D", "env.java.opts=-DappName=foobar", "-D", "security.ssl.internal.key-password=changeit"});
- YarnClusterDescriptor flinkYarnDescriptor = cli.createClusterDescriptor(cmd);
+ Configuration executorConfig = cli.applyCommandLineOptionsToConfiguration(cmd);
+ YarnClusterDescriptor flinkYarnDescriptor = cli.createClusterDescriptor(executorConfig);
Assert.assertNotNull(flinkYarnDescriptor);
@@ -109,10 +110,10 @@ public class FlinkYarnSessionCliTest extends TestLogger {
"yarn");
final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);
+ final Configuration executorConfig = yarnCLI.applyCommandLineOptionsToConfiguration(commandLine);
- YarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);
-
- final ClusterSpecification clusterSpecification = yarnCLI.getClusterSpecification(commandLine);
+ final YarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(executorConfig);
+ final ClusterSpecification clusterSpecification = yarnCLI.getClusterSpecification(executorConfig);
// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
assertEquals(3, clusterSpecification.getSlotsPerTaskManager());
@@ -131,8 +132,9 @@ public class FlinkYarnSessionCliTest extends TestLogger {
"yarn");
final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);
+ final Configuration executorConfig = yarnCLI.applyCommandLineOptionsToConfiguration(commandLine);
- YarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);
+ YarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(executorConfig);
// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
assertTrue(descriptor.isDetachedMode());
@@ -151,8 +153,9 @@ public class FlinkYarnSessionCliTest extends TestLogger {
"yarn");
CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);
+ Configuration executorConfig = yarnCLI.applyCommandLineOptionsToConfiguration(commandLine);
- YarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);
+ YarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(executorConfig);
assertEquals(zkNamespaceCliInput, descriptor.getZookeeperNamespace());
}
@@ -170,8 +173,9 @@ public class FlinkYarnSessionCliTest extends TestLogger {
"yarn");
CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);
+ Configuration executorConfig = yarnCLI.applyCommandLineOptionsToConfiguration(commandLine);
- YarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);
+ YarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(executorConfig);
assertEquals(nodeLabelCliInput, descriptor.getNodeLabel());
}
@@ -194,8 +198,9 @@ public class FlinkYarnSessionCliTest extends TestLogger {
"yarn");
final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {}, true);
+ final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
- final ApplicationId clusterId = flinkYarnSessionCli.getClusterId(commandLine);
+ final ApplicationId clusterId = flinkYarnSessionCli.getClusterId(executorConfig);
assertEquals(TEST_YARN_APPLICATION_ID, clusterId);
}
@@ -229,8 +234,9 @@ public class FlinkYarnSessionCliTest extends TestLogger {
"yarn");
final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString()}, true);
+ final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
- final ApplicationId clusterId = flinkYarnSessionCli.getClusterId(commandLine);
+ final ApplicationId clusterId = flinkYarnSessionCli.getClusterId(executorConfig);
assertEquals(TEST_YARN_APPLICATION_ID, clusterId);
}
@@ -245,8 +251,9 @@ public class FlinkYarnSessionCliTest extends TestLogger {
"yarn");
final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString()}, true);
+ final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
- final YarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);
+ final YarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(executorConfig);
final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();
@@ -266,8 +273,9 @@ public class FlinkYarnSessionCliTest extends TestLogger {
final String overrideZkNamespace = "my_cluster";
final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true);
+ final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
- final YarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);
+ final YarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(executorConfig);
final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();
@@ -288,7 +296,9 @@ public class FlinkYarnSessionCliTest extends TestLogger {
"y",
"yarn");
final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID_2.toString() }, true);
- final ApplicationId clusterId = flinkYarnSessionCli.getClusterId(commandLine);
+ final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
+
+ final ApplicationId clusterId = flinkYarnSessionCli.getClusterId(executorConfig);
assertEquals(TEST_YARN_APPLICATION_ID_2, clusterId);
}
@@ -315,8 +325,9 @@ public class FlinkYarnSessionCliTest extends TestLogger {
"yarn");
CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);
+ Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
- final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);
+ final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(executorConfig);
assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
@@ -345,8 +356,9 @@ public class FlinkYarnSessionCliTest extends TestLogger {
"yarn");
CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);
+ Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
- final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);
+ final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(executorConfig);
assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
@@ -366,8 +378,9 @@ public class FlinkYarnSessionCliTest extends TestLogger {
"yarn");
final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);
+ final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
- final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);
+ final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(executorConfig);
assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
@@ -385,7 +398,9 @@ public class FlinkYarnSessionCliTest extends TestLogger {
"y",
"yarn");
final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);
- final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);
+ final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
+
+ final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(executorConfig);
assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
@@ -403,7 +418,9 @@ public class FlinkYarnSessionCliTest extends TestLogger {
"y",
"yarn");
final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);
- final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);
+ final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
+
+ final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(executorConfig);
assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
@@ -425,8 +442,9 @@ public class FlinkYarnSessionCliTest extends TestLogger {
"yarn");
final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);
+ final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
- final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);
+ final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(executorConfig);
assertThat(clusterSpecification.getMasterMemoryMB(), is(2048));
assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(4096));
@@ -444,8 +462,9 @@ public class FlinkYarnSessionCliTest extends TestLogger {
"yarn");
final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);
+ final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
- final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);
+ final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(executorConfig);
assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(1024));
@@ -461,8 +480,9 @@ public class FlinkYarnSessionCliTest extends TestLogger {
"yarn");
final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);
+ final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
- YarnClusterDescriptor flinkYarnDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);
+ YarnClusterDescriptor flinkYarnDescriptor = flinkYarnSessionCli.createClusterDescriptor(executorConfig);
assertEquals(2, flinkYarnDescriptor.getShipFiles().size());
diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java
index 143749b04..69af7c3 100644
--- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java
+++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java
@@ -525,7 +525,6 @@ public class YarnClusterDescriptorTest extends TestLogger {
yarnClusterDescriptor = new YarnClusterDescriptor(
new Configuration(),
yarnConfiguration,
- temporaryFolder.getRoot().getAbsolutePath(),
closableYarnClient,
false);
@@ -542,7 +541,6 @@ public class YarnClusterDescriptorTest extends TestLogger {
return new YarnClusterDescriptor(
configuration,
yarnConfiguration,
- temporaryFolder.getRoot().getAbsolutePath(),
yarnClient,
true);
}