You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by kk...@apache.org on 2019/10/25 14:47:58 UTC
[flink] 08/08: [FLINK-14377] Parse the ProgramOptions to a
Configuration.
This is an automated email from the ASF dual-hosted git repository.
kkloudas pushed a commit to branch to-merge
in repository https://gitbox.apache.org/repos/asf/flink.git
commit c5dc8eae5c2b21d57bec0aef9e6b79d7af9b8eff
Author: Kostas Kloudas <kk...@gmail.com>
AuthorDate: Thu Oct 10 11:39:51 2019 +0200
[FLINK-14377] Parse the ProgramOptions to a Configuration.
---
.../generated/deployment_configuration.html | 5 +
.../generated/pipeline_configuration.html | 21 ++++
...on.html => savepoint_config_configuration.html} | 8 +-
docs/ops/config.md | 5 +
docs/ops/config.zh.md | 5 +
.../org/apache/flink/client/cli/CliFrontend.java | 65 ++++++-----
.../apache/flink/client/cli/CliFrontendParser.java | 13 +--
.../flink/client/cli/ExecutionConfigAccessor.java | 127 +++++++++++++++++++++
.../client/cli/ExecutionConfigurationUtils.java | 96 ++++++++++++++++
.../org/apache/flink/client/cli/InfoOptions.java | 31 -----
.../apache/flink/client/cli/ProgramOptions.java | 7 +-
.../org/apache/flink/client/cli/RunOptions.java | 31 -----
.../client/cli/CliFrontendPackageProgramTest.java | 93 +++++++++------
.../flink/client/cli/CliFrontendRunTest.java | 32 ++++--
.../flink/configuration/DeploymentOptions.java | 6 +
...DeploymentOptions.java => PipelineOptions.java} | 27 +++--
.../configuration/ConfigOptionsDocGenerator.java | 1 +
.../runtime/jobgraph/SavepointConfigOptions.java | 27 +++--
.../runtime/jobgraph/SavepointRestoreSettings.java | 17 +++
.../client/gateway/local/ExecutionContext.java | 16 +--
20 files changed, 449 insertions(+), 184 deletions(-)
diff --git a/docs/_includes/generated/deployment_configuration.html b/docs/_includes/generated/deployment_configuration.html
index 2c652e4..7f7bdcc 100644
--- a/docs/_includes/generated/deployment_configuration.html
+++ b/docs/_includes/generated/deployment_configuration.html
@@ -13,6 +13,11 @@
<td>Specifies if the pipeline is submitted in attached or detached mode.</td>
</tr>
<tr>
+ <td><h5>execution.shutdown-on-attached-exit</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>If the job is submitted in attached mode, perform a best-effort cluster shutdown when the CLI is terminated abruptly, e.g., in response to a user interrupt, such as typing Ctrl + C.</td>
+ </tr>
+ <tr>
<td><h5>execution.target</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>The deployment target for the execution, e.g. "local" for local execution.</td>
diff --git a/docs/_includes/generated/pipeline_configuration.html b/docs/_includes/generated/pipeline_configuration.html
new file mode 100644
index 0000000..36cf711
--- /dev/null
+++ b/docs/_includes/generated/pipeline_configuration.html
@@ -0,0 +1,21 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 65%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>pipeline.classpaths</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>A semicolon-separated list of the classpaths to package with the job jars to be sent to the cluster. These have to be valid URLs.</td>
+ </tr>
+ <tr>
+ <td><h5>pipeline.jars</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>A semicolon-separated list of the jars to package with the job jars to be sent to the cluster. These have to be valid paths.</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/deployment_configuration.html b/docs/_includes/generated/savepoint_config_configuration.html
similarity index 54%
copy from docs/_includes/generated/deployment_configuration.html
copy to docs/_includes/generated/savepoint_config_configuration.html
index 2c652e4..10318f4 100644
--- a/docs/_includes/generated/deployment_configuration.html
+++ b/docs/_includes/generated/savepoint_config_configuration.html
@@ -8,14 +8,14 @@
</thead>
<tbody>
<tr>
- <td><h5>execution.attached</h5></td>
+ <td><h5>execution.savepoint.ignore-unclaimed-state</h5></td>
<td style="word-wrap: break-word;">false</td>
- <td>Specifies if the pipeline is submitted in attached or detached mode.</td>
+ <td>Allow to skip savepoint state that cannot be restored. Allow this if you removed an operator from your pipeline after the savepoint was triggered.</td>
</tr>
<tr>
- <td><h5>execution.target</h5></td>
+ <td><h5>execution.savepoint.path</h5></td>
<td style="word-wrap: break-word;">(none)</td>
- <td>The deployment target for the execution, e.g. "local" for local execution.</td>
+ <td>Path to a savepoint to restore the job from (for example hdfs:///flink/savepoint-1537).</td>
</tr>
</tbody>
</table>
diff --git a/docs/ops/config.md b/docs/ops/config.md
index 6a8f4ce..9390dbd 100644
--- a/docs/ops/config.md
+++ b/docs/ops/config.md
@@ -67,6 +67,7 @@ These parameters configure the default HDFS used by Flink. Setups that do not sp
### Execution
{% include generated/deployment_configuration.html %}
+{% include generated/savepoint_config_configuration.html %}
### JobManager
@@ -188,6 +189,10 @@ The configuration keys in this section are independent of the used resource mana
{% include generated/environment_configuration.html %}
+### Pipeline
+
+{% include generated/pipeline_configuration.html %}
+
### Checkpointing
{% include generated/checkpointing_configuration.html %}
diff --git a/docs/ops/config.zh.md b/docs/ops/config.zh.md
index 1274f6d..d3faea0 100644
--- a/docs/ops/config.zh.md
+++ b/docs/ops/config.zh.md
@@ -67,6 +67,7 @@ These parameters configure the default HDFS used by Flink. Setups that do not sp
### Execution
{% include generated/deployment_configuration.html %}
+{% include generated/savepoint_config_configuration.html %}
### JobManager
@@ -188,6 +189,10 @@ The configuration keys in this section are independent of the used resource mana
{% include generated/environment_configuration.html %}
+### Pipeline
+
+{% include generated/pipeline_configuration.html %}
+
### Checkpointing
{% include generated/checkpointing_configuration.html %}
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
index e1a838f..b3a7860 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
@@ -56,7 +56,6 @@ import org.apache.flink.runtime.security.SecurityUtils;
import org.apache.flink.runtime.util.EnvironmentInformation;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.FlinkException;
-import org.apache.flink.util.Preconditions;
import org.apache.flink.util.ShutdownHookUtil;
import org.apache.commons.cli.CommandLine;
@@ -84,6 +83,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
+import static org.apache.flink.client.cli.CliFrontendParser.HELP_OPTION;
import static org.apache.flink.util.Preconditions.checkNotNull;
import static org.apache.flink.util.Preconditions.checkState;
@@ -186,17 +186,18 @@ public class CliFrontend {
final CommandLine commandLine = CliFrontendParser.parse(commandLineOptions, args, true);
- final RunOptions runOptions = new RunOptions(commandLine);
+ final ProgramOptions programOptions = new ProgramOptions(commandLine);
+ final ExecutionConfigAccessor executionParameters = ExecutionConfigAccessor.fromProgramOptions(programOptions);
// evaluate help flag
- if (runOptions.isPrintHelp()) {
+ if (commandLine.hasOption(HELP_OPTION.getOpt())) {
CliFrontendParser.printHelpForRun(customCommandLines);
return;
}
- if (!runOptions.isPython()) {
+ if (!programOptions.isPython()) {
// Java program should be specified a JAR file
- if (runOptions.getJarFilePath() == null) {
+ if (executionParameters.getJarFilePath() == null) {
throw new CliArgsException("Java program should be specified a JAR file.");
}
}
@@ -204,7 +205,7 @@ public class CliFrontend {
final PackagedProgram program;
try {
LOG.info("Building program from JAR file");
- program = buildProgram(runOptions);
+ program = buildProgram(programOptions, executionParameters);
}
catch (FileNotFoundException e) {
throw new CliArgsException("Could not build the program from JAR file.", e);
@@ -212,8 +213,9 @@ public class CliFrontend {
final CustomCommandLine customCommandLine = getActiveCustomCommandLine(commandLine);
final Configuration executorConfig = customCommandLine.applyCommandLineOptionsToConfiguration(commandLine);
+ final Configuration executionConfig = executionParameters.getConfiguration();
try {
- runProgram(executorConfig, runOptions, program);
+ runProgram(executorConfig, executionConfig, program);
} finally {
program.deleteExtractedLibraries();
}
@@ -221,7 +223,7 @@ public class CliFrontend {
private <T> void runProgram(
Configuration executorConfig,
- RunOptions runOptions,
+ Configuration executionConfig,
PackagedProgram program) throws ProgramInvocationException, FlinkException {
final ClusterClientFactory<T> clusterClientFactory = clusterClientServiceLoader.getClusterClientFactory(executorConfig);
@@ -231,12 +233,12 @@ public class CliFrontend {
try {
final T clusterId = clusterClientFactory.getClusterId(executorConfig);
-
+ final ExecutionConfigAccessor executionParameters = ExecutionConfigAccessor.fromConfiguration(executionConfig);
final ClusterClient<T> client;
// directly deploy the job if the cluster is started in job mode and detached
- if (clusterId == null && runOptions.getDetachedMode()) {
- int parallelism = runOptions.getParallelism() == -1 ? defaultParallelism : runOptions.getParallelism();
+ if (clusterId == null && executionParameters.getDetachedMode()) {
+ int parallelism = executionParameters.getParallelism() == -1 ? defaultParallelism : executionParameters.getParallelism();
final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(program, configuration, parallelism);
@@ -244,7 +246,7 @@ public class CliFrontend {
client = clusterDescriptor.deployJobCluster(
clusterSpecification,
jobGraph,
- runOptions.getDetachedMode());
+ executionParameters.getDetachedMode());
logAndSysout("Job has been submitted with JobID " + jobGraph.getJobID());
@@ -265,7 +267,7 @@ public class CliFrontend {
client = clusterDescriptor.deploySessionCluster(clusterSpecification);
// if not running in detached mode, add a shutdown hook to shut down cluster if client exits
// there's a race-condition here if cli is killed before shutdown hook is installed
- if (!runOptions.getDetachedMode() && runOptions.isShutdownOnAttachedExit()) {
+ if (!executionParameters.getDetachedMode() && executionParameters.isShutdownOnAttachedExit()) {
shutdownHook = ShutdownHookUtil.addShutdownHook(client::shutDownCluster, client.getClass().getSimpleName(), LOG);
} else {
shutdownHook = null;
@@ -273,11 +275,9 @@ public class CliFrontend {
}
try {
- client.setDetached(runOptions.getDetachedMode());
-
- LOG.debug("{}", runOptions.getSavepointRestoreSettings());
+ client.setDetached(executionParameters.getDetachedMode());
- int userParallelism = runOptions.getParallelism();
+ int userParallelism = executionParameters.getParallelism();
LOG.debug("User parallelism is set to {}", userParallelism);
if (ExecutionConfig.PARALLELISM_DEFAULT == userParallelism) {
userParallelism = defaultParallelism;
@@ -325,25 +325,26 @@ public class CliFrontend {
final CommandLine commandLine = CliFrontendParser.parse(commandOptions, args, true);
- InfoOptions infoOptions = new InfoOptions(commandLine);
+ final ProgramOptions programOptions = new ProgramOptions(commandLine);
+ final ExecutionConfigAccessor executionParameters = ExecutionConfigAccessor.fromProgramOptions(programOptions);
// evaluate help flag
- if (infoOptions.isPrintHelp()) {
+ if (commandLine.hasOption(HELP_OPTION.getOpt())) {
CliFrontendParser.printHelpForInfo();
return;
}
- if (infoOptions.getJarFilePath() == null) {
+ if (programOptions.getJarFilePath() == null) {
throw new CliArgsException("The program JAR file was not specified.");
}
// -------- build the packaged program -------------
LOG.info("Building program from JAR file");
- final PackagedProgram program = buildProgram(infoOptions);
+ final PackagedProgram program = buildProgram(programOptions, executionParameters);
try {
- int parallelism = infoOptions.getParallelism();
+ int parallelism = programOptions.getParallelism();
if (ExecutionConfig.PARALLELISM_DEFAULT == parallelism) {
parallelism = defaultParallelism;
}
@@ -724,7 +725,7 @@ public class CliFrontend {
* Sends a SavepointDisposalRequest to the job manager.
*/
private void disposeSavepoint(ClusterClient<?> clusterClient, String savepointPath) throws FlinkException {
- Preconditions.checkNotNull(savepointPath, "Missing required argument: savepoint path. " +
+ checkNotNull(savepointPath, "Missing required argument: savepoint path. " +
"Usage: bin/flink savepoint -d <savepoint-path>");
logAndSysout("Disposing savepoint '" + savepointPath + "'.");
@@ -771,15 +772,17 @@ public class CliFrontend {
*
* @return A PackagedProgram (upon success)
*/
- PackagedProgram buildProgram(ProgramOptions options) throws FileNotFoundException, ProgramInvocationException {
- String[] programArgs = options.getProgramArgs();
- String jarFilePath = options.getJarFilePath();
- List<URL> classpaths = options.getClasspaths();
+ PackagedProgram buildProgram(
+ final ProgramOptions runOptions,
+ final ExecutionConfigAccessor executionParameters) throws FileNotFoundException, ProgramInvocationException {
+ String[] programArgs = runOptions.getProgramArgs();
+ String jarFilePath = executionParameters.getJarFilePath();
+ List<URL> classpaths = executionParameters.getClasspaths();
// Get assembler class
- String entryPointClass = options.getEntryPointClassName();
+ String entryPointClass = runOptions.getEntryPointClassName();
File jarFile = null;
- if (options.isPython()) {
+ if (runOptions.isPython()) {
// If the job is specified a jar file
if (jarFilePath != null) {
jarFile = getJarFile(jarFilePath);
@@ -801,7 +804,7 @@ public class CliFrontend {
new PackagedProgram(jarFile, classpaths, programArgs) :
new PackagedProgram(jarFile, classpaths, entryPointClass, programArgs);
- program.setSavepointRestoreSettings(options.getSavepointRestoreSettings());
+ program.setSavepointRestoreSettings(executionParameters.getSavepointRestoreSettings());
return program;
}
@@ -1192,7 +1195,7 @@ public class CliFrontend {
// construct class types from the parameters
Class<?>[] types = new Class<?>[params.length];
for (int i = 0; i < params.length; i++) {
- Preconditions.checkNotNull(params[i], "Parameters for custom command-lines may not be null.");
+ checkNotNull(params[i], "Parameters for custom command-lines may not be null.");
types[i] = params[i].getClass();
}
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java
index 97ba33e..c0295c7 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java
@@ -192,7 +192,7 @@ public class CliFrontendParser {
PYMODULE_OPTION.setArgName("pyModule");
}
- private static final Options RUN_OPTIONS = getRunCommandOptions();
+ static final Options RUN_OPTIONS = getRunCommandOptions();
private static Options buildGeneralOptions(Options options) {
options.addOption(HELP_OPTION);
@@ -445,17 +445,6 @@ public class CliFrontendParser {
// Line Parsing
// --------------------------------------------------------------------------------------------
- public static RunOptions parseRunCommand(String[] args) throws CliArgsException {
- try {
- DefaultParser parser = new DefaultParser();
- CommandLine line = parser.parse(RUN_OPTIONS, args, true);
- return new RunOptions(line);
- }
- catch (ParseException e) {
- throw new CliArgsException(e.getMessage());
- }
- }
-
public static CommandLine parse(Options options, String[] args, boolean stopAtNonOptions) throws CliArgsException {
final DefaultParser parser = new DefaultParser();
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutionConfigAccessor.java b/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutionConfigAccessor.java
new file mode 100644
index 0000000..194a352
--- /dev/null
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutionConfigAccessor.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.cli;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.DeploymentOptions;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Accessor that exposes config settings that are relevant for execution from an underlying {@link Configuration}.
+ */
+@Internal
+public class ExecutionConfigAccessor {
+
+ private final Configuration configuration;
+
+ private ExecutionConfigAccessor(final Configuration configuration) {
+ this.configuration = checkNotNull(configuration);
+ }
+
+ /**
+ * Creates an {@link ExecutionConfigAccessor} based on the provided {@link Configuration}.
+ */
+ public static ExecutionConfigAccessor fromConfiguration(final Configuration configuration) {
+ return new ExecutionConfigAccessor(checkNotNull(configuration));
+ }
+
+ /**
+ * Creates an {@link ExecutionConfigAccessor} based on the provided {@link ProgramOptions} as provided by the user through the CLI.
+ */
+ public static ExecutionConfigAccessor fromProgramOptions(final ProgramOptions options) {
+ checkNotNull(options);
+
+ final Configuration configuration = new Configuration();
+ configuration.setInteger(CoreOptions.DEFAULT_PARALLELISM, options.getParallelism());
+ configuration.setBoolean(DeploymentOptions.ATTACHED, !options.getDetachedMode());
+ configuration.setBoolean(DeploymentOptions.SHUTDOWN_IF_ATTACHED, options.isShutdownOnAttachedExit());
+
+ parseClasspathURLsToConfig(options.getClasspaths(), configuration);
+ parseJarURLToConfig(options.getJarFilePath(), configuration);
+
+ SavepointRestoreSettings.toConfiguration(options.getSavepointRestoreSettings(), configuration);
+
+ return new ExecutionConfigAccessor(configuration);
+ }
+
+ private static void parseClasspathURLsToConfig(final List<URL> classpathURLs, final Configuration configuration) {
+ ExecutionConfigurationUtils.urlListToConfig(
+ classpathURLs,
+ PipelineOptions.LIST_SEPARATOR,
+ configuration,
+ PipelineOptions.CLASSPATHS);
+ }
+
+ private static void parseJarURLToConfig(final String jarFile, final Configuration configuration) {
+ if (jarFile == null) {
+ return;
+ }
+
+ try {
+ final URL jarUrl = new File(jarFile).getAbsoluteFile().toURI().toURL();
+ final List<URL> jarUrlSingleton = Collections.singletonList(jarUrl);
+ ExecutionConfigurationUtils.urlListToConfig(jarUrlSingleton, PipelineOptions.LIST_SEPARATOR, configuration, PipelineOptions.JARS);
+ } catch (MalformedURLException e) {
+ throw new IllegalArgumentException("JAR file path invalid", e);
+ }
+ }
+
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public String getJarFilePath() {
+ final List<URL> jarURL = ExecutionConfigurationUtils.urlListFromConfig(configuration, PipelineOptions.JARS, PipelineOptions.LIST_SEPARATOR);
+ if (jarURL != null && !jarURL.isEmpty()) {
+ return jarURL.get(0).getPath();
+ }
+ return null;
+ }
+
+ public List<URL> getClasspaths() {
+ return ExecutionConfigurationUtils.urlListFromConfig(configuration, PipelineOptions.CLASSPATHS, PipelineOptions.LIST_SEPARATOR);
+ }
+
+ public int getParallelism() {
+ return configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM);
+ }
+
+ public boolean getDetachedMode() {
+ return !configuration.getBoolean(DeploymentOptions.ATTACHED);
+ }
+
+ public SavepointRestoreSettings getSavepointRestoreSettings() {
+ return SavepointRestoreSettings.fromConfiguration(configuration);
+ }
+
+ public boolean isShutdownOnAttachedExit() {
+ return configuration.getBoolean(DeploymentOptions.SHUTDOWN_IF_ATTACHED);
+ }
+}
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutionConfigurationUtils.java b/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutionConfigurationUtils.java
new file mode 100644
index 0000000..099984a
--- /dev/null
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutionConfigurationUtils.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.cli;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Utilities for parsing parameters in the {@link ExecutionConfigAccessor}.
+ */
+@Internal
+class ExecutionConfigurationUtils {
+
+ /**
+ * Parses a list of {@link URL URLs} to a string and puts it in the provided {@code configuration} as the value of the provided {@code option}.
+ * @param urls the list of URLs to parse
+ * @param delimiter the delimiter to be used to separate the members of the list in the string
+ * @param configuration the configuration object to put the list
+ * @param option the {@link ConfigOption option} to serve as the key for the list in the configuration
+ * @return the produced string to be put in the configuration.
+ */
+ static String urlListToConfig(
+ final List<URL> urls,
+ final String delimiter,
+ final Configuration configuration,
+ final ConfigOption<String> option) {
+
+ checkNotNull(urls);
+ checkNotNull(delimiter);
+ checkNotNull(configuration);
+ checkNotNull(option);
+
+ final String str = urls.stream().map(URL::toString).collect(Collectors.joining(delimiter));
+ configuration.setString(option, str);
+ return str;
+ }
+
+ /**
+ * Parses a string into a list of {@link URL URLs} from a given {@link Configuration}.
+ * @param configuration the configuration containing the string-ified list of URLs
+ * @param option the {@link ConfigOption option} whose value is the list of URLs
+ * @param delimiter the delimiter used to separate the members of the list in the string
+ * @return the produced list of URLs.
+ */
+ static List<URL> urlListFromConfig(
+ final Configuration configuration,
+ final ConfigOption<String> option,
+ final String delimiter) {
+
+ checkNotNull(configuration);
+ checkNotNull(option);
+ checkNotNull(delimiter);
+
+ final String urls = configuration.getString(option);
+ if (urls == null || urls.length() == 0) {
+ return Collections.emptyList();
+ }
+
+ try (final Stream<String> urlTokens = Arrays.stream(urls.split(delimiter))) {
+ return urlTokens.map(str -> {
+ try {
+ return new URL(str);
+ } catch (MalformedURLException e) {
+ throw new IllegalArgumentException("Invalid URL", e);
+ }
+ }).collect(Collectors.toList());
+ }
+ }
+}
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/InfoOptions.java b/flink-clients/src/main/java/org/apache/flink/client/cli/InfoOptions.java
deleted file mode 100644
index 559ce94..0000000
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/InfoOptions.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.client.cli;
-
-import org.apache.commons.cli.CommandLine;
-
-/**
- * Command line options for the INFO command.
- */
-public class InfoOptions extends ProgramOptions {
-
- public InfoOptions(CommandLine line) throws CliArgsException {
- super(line);
- }
-}
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java b/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java
index e3a9907..be3b811 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java
@@ -44,7 +44,7 @@ import static org.apache.flink.client.cli.CliFrontendParser.YARN_DETACHED_OPTION
/**
* Base class for command line options that refer to a JAR file program.
*/
-public abstract class ProgramOptions extends CommandLineOptions {
+public class ProgramOptions extends CommandLineOptions {
private final String jarFilePath;
@@ -67,7 +67,7 @@ public abstract class ProgramOptions extends CommandLineOptions {
*/
private final boolean isPython;
- protected ProgramOptions(CommandLine line) throws CliArgsException {
+ public ProgramOptions(CommandLine line) throws CliArgsException {
super(line);
String[] args = line.hasOption(ARGS_OPTION.getOpt()) ?
@@ -168,8 +168,7 @@ public abstract class ProgramOptions extends CommandLineOptions {
parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
}
- detachedMode = line.hasOption(DETACHED_OPTION.getOpt()) || line.hasOption(
- YARN_DETACHED_OPTION.getOpt());
+ detachedMode = line.hasOption(DETACHED_OPTION.getOpt()) || line.hasOption(YARN_DETACHED_OPTION.getOpt());
shutdownOnAttachedExit = line.hasOption(SHUTDOWN_IF_ATTACHED_OPTION.getOpt());
this.savepointSettings = CliFrontendParser.createSavepointRestoreSettings(line);
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/RunOptions.java b/flink-clients/src/main/java/org/apache/flink/client/cli/RunOptions.java
deleted file mode 100644
index 08a15d3..0000000
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/RunOptions.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.client.cli;
-
-import org.apache.commons.cli.CommandLine;
-
-/**
- * Command line options for the RUN command.
- */
-public class RunOptions extends ProgramOptions {
-
- public RunOptions(CommandLine line) throws CliArgsException {
- super(line);
- }
-}
diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendPackageProgramTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendPackageProgramTest.java
index 873ba00..2efb8ca 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendPackageProgramTest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendPackageProgramTest.java
@@ -29,6 +29,7 @@ import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.costs.DefaultCostEstimator;
import org.apache.flink.util.TestLogger;
+import org.apache.commons.cli.CommandLine;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
@@ -78,11 +79,12 @@ public class CliFrontendPackageProgramTest extends TestLogger {
@Test
public void testNonExistingJarFile() throws Exception {
- ProgramOptions options = mock(ProgramOptions.class);
- when(options.getJarFilePath()).thenReturn("/some/none/existing/path");
+ ProgramOptions programOptions = mock(ProgramOptions.class);
+ ExecutionConfigAccessor executionOptions = mock(ExecutionConfigAccessor.class);
+ when(executionOptions.getJarFilePath()).thenReturn("/some/none/existing/path");
try {
- frontend.buildProgram(options);
+ frontend.buildProgram(programOptions, executionOptions);
fail("should throw an exception");
}
catch (FileNotFoundException e) {
@@ -92,11 +94,12 @@ public class CliFrontendPackageProgramTest extends TestLogger {
@Test
public void testFileNotJarFile() throws Exception {
- ProgramOptions options = mock(ProgramOptions.class);
- when(options.getJarFilePath()).thenReturn(getNonJarFilePath());
+ ProgramOptions programOptions = mock(ProgramOptions.class);
+ ExecutionConfigAccessor executionOptions = mock(ExecutionConfigAccessor.class);
+ when(executionOptions.getJarFilePath()).thenReturn(getNonJarFilePath());
try {
- frontend.buildProgram(options);
+ frontend.buildProgram(programOptions, executionOptions);
fail("should throw an exception");
}
catch (ProgramInvocationException e) {
@@ -114,12 +117,15 @@ public class CliFrontendPackageProgramTest extends TestLogger {
URL[] classpath = new URL[] { new URL("file:///tmp/foo"), new URL("file:///tmp/bar") };
String[] reducedArguments = new String[] {"--debug", "true", "arg1", "arg2"};
- RunOptions options = CliFrontendParser.parseRunCommand(arguments);
- assertEquals(getTestJarPath(), options.getJarFilePath());
- assertArrayEquals(classpath, options.getClasspaths().toArray());
- assertArrayEquals(reducedArguments, options.getProgramArgs());
+ CommandLine commandLine = CliFrontendParser.parse(CliFrontendParser.RUN_OPTIONS, arguments, true);
+ ProgramOptions programOptions = new ProgramOptions(commandLine);
+ ExecutionConfigAccessor executionOptions = ExecutionConfigAccessor.fromProgramOptions(programOptions);
- PackagedProgram prog = frontend.buildProgram(options);
+ assertEquals(getTestJarPath(), executionOptions.getJarFilePath());
+ assertArrayEquals(classpath, executionOptions.getClasspaths().toArray());
+ assertArrayEquals(reducedArguments, programOptions.getProgramArgs());
+
+ PackagedProgram prog = frontend.buildProgram(programOptions, executionOptions);
Assert.assertArrayEquals(reducedArguments, prog.getArguments());
Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName());
@@ -135,12 +141,15 @@ public class CliFrontendPackageProgramTest extends TestLogger {
URL[] classpath = new URL[] { new URL("file:///tmp/foo"), new URL("file:///tmp/bar") };
String[] reducedArguments = new String[] {"--debug", "true", "arg1", "arg2"};
- RunOptions options = CliFrontendParser.parseRunCommand(arguments);
- assertEquals(getTestJarPath(), options.getJarFilePath());
- assertArrayEquals(classpath, options.getClasspaths().toArray());
- assertArrayEquals(reducedArguments, options.getProgramArgs());
+ CommandLine commandLine = CliFrontendParser.parse(CliFrontendParser.RUN_OPTIONS, arguments, true);
+ ProgramOptions programOptions = new ProgramOptions(commandLine);
+ ExecutionConfigAccessor executionOptions = ExecutionConfigAccessor.fromProgramOptions(programOptions);
+
+ assertEquals(getTestJarPath(), executionOptions.getJarFilePath());
+ assertArrayEquals(classpath, executionOptions.getClasspaths().toArray());
+ assertArrayEquals(reducedArguments, programOptions.getProgramArgs());
- PackagedProgram prog = frontend.buildProgram(options);
+ PackagedProgram prog = frontend.buildProgram(programOptions, executionOptions);
Assert.assertArrayEquals(reducedArguments, prog.getArguments());
Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName());
@@ -156,12 +165,15 @@ public class CliFrontendPackageProgramTest extends TestLogger {
URL[] classpath = new URL[] { new URL("file:///tmp/foo"), new URL("file:///tmp/bar") };
String[] reducedArguments = {"--debug", "true", "arg1", "arg2"};
- RunOptions options = CliFrontendParser.parseRunCommand(arguments);
- assertEquals(getTestJarPath(), options.getJarFilePath());
- assertArrayEquals(classpath, options.getClasspaths().toArray());
- assertArrayEquals(reducedArguments, options.getProgramArgs());
+ CommandLine commandLine = CliFrontendParser.parse(CliFrontendParser.RUN_OPTIONS, arguments, true);
+ ProgramOptions programOptions = new ProgramOptions(commandLine);
+ ExecutionConfigAccessor executionOptions = ExecutionConfigAccessor.fromProgramOptions(programOptions);
+
+ assertEquals(getTestJarPath(), executionOptions.getJarFilePath());
+ assertArrayEquals(classpath, executionOptions.getClasspaths().toArray());
+ assertArrayEquals(reducedArguments, programOptions.getProgramArgs());
- PackagedProgram prog = frontend.buildProgram(options);
+ PackagedProgram prog = frontend.buildProgram(programOptions, executionOptions);
Assert.assertArrayEquals(reducedArguments, prog.getArguments());
Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName());
@@ -182,13 +194,16 @@ public class CliFrontendPackageProgramTest extends TestLogger {
URL[] classpath = new URL[] { new URL("file:///tmp/foo"), new URL("file:///tmp/bar") };
String[] reducedArguments = {"--debug", "true", "arg1", "arg2"};
- RunOptions options = CliFrontendParser.parseRunCommand(arguments);
- assertEquals(arguments[4], options.getJarFilePath());
- assertArrayEquals(classpath, options.getClasspaths().toArray());
- assertArrayEquals(reducedArguments, options.getProgramArgs());
+ CommandLine commandLine = CliFrontendParser.parse(CliFrontendParser.RUN_OPTIONS, arguments, true);
+ ProgramOptions programOptions = new ProgramOptions(commandLine);
+ ExecutionConfigAccessor executionOptions = ExecutionConfigAccessor.fromProgramOptions(programOptions);
+
+ assertEquals(arguments[4], executionOptions.getJarFilePath());
+ assertArrayEquals(classpath, executionOptions.getClasspaths().toArray());
+ assertArrayEquals(reducedArguments, programOptions.getProgramArgs());
try {
- frontend.buildProgram(options);
+ frontend.buildProgram(programOptions, executionOptions);
fail("Should fail with an exception");
}
catch (FileNotFoundException e) {
@@ -200,12 +215,15 @@ public class CliFrontendPackageProgramTest extends TestLogger {
public void testNonExistingFileWithoutArguments() throws Exception {
String[] arguments = {"/some/none/existing/path"};
- RunOptions options = CliFrontendParser.parseRunCommand(arguments);
- assertEquals(arguments[0], options.getJarFilePath());
- assertArrayEquals(new String[0], options.getProgramArgs());
+ CommandLine commandLine = CliFrontendParser.parse(CliFrontendParser.RUN_OPTIONS, arguments, true);
+ ProgramOptions programOptions = new ProgramOptions(commandLine);
+ ExecutionConfigAccessor executionOptions = ExecutionConfigAccessor.fromProgramOptions(programOptions);
+
+ assertEquals(arguments[0], executionOptions.getJarFilePath());
+ assertArrayEquals(new String[0], programOptions.getProgramArgs());
try {
- frontend.buildProgram(options);
+ frontend.buildProgram(programOptions, executionOptions);
}
catch (FileNotFoundException e) {
// that's what we want
@@ -258,13 +276,16 @@ public class CliFrontendPackageProgramTest extends TestLogger {
URL[] classpath = new URL[] { new URL("file:///tmp/foo"), new URL("file:///tmp/bar") };
String[] reducedArguments = { "true", "arg1", "arg2" };
- RunOptions options = CliFrontendParser.parseRunCommand(arguments);
- assertEquals(getTestJarPath(), options.getJarFilePath());
- assertArrayEquals(classpath, options.getClasspaths().toArray());
- assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, options.getEntryPointClassName());
- assertArrayEquals(reducedArguments, options.getProgramArgs());
+ CommandLine commandLine = CliFrontendParser.parse(CliFrontendParser.RUN_OPTIONS, arguments, true);
+ ProgramOptions programOptions = new ProgramOptions(commandLine);
+ ExecutionConfigAccessor executionOptions = ExecutionConfigAccessor.fromProgramOptions(programOptions);
+
+ assertEquals(getTestJarPath(), executionOptions.getJarFilePath());
+ assertArrayEquals(classpath, executionOptions.getClasspaths().toArray());
+ assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, programOptions.getEntryPointClassName());
+ assertArrayEquals(reducedArguments, programOptions.getProgramArgs());
- PackagedProgram prog = spy(frontend.buildProgram(options));
+ PackagedProgram prog = spy(frontend.buildProgram(programOptions, executionOptions));
ClassLoader testClassLoader = new ClassLoader(prog.getUserCodeClassLoader()) {
@Override
diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendRunTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendRunTest.java
index 2230917..918b6db 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendRunTest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendRunTest.java
@@ -25,6 +25,7 @@ import org.apache.flink.client.program.PackagedProgram;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+import org.apache.commons.cli.CommandLine;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -81,8 +82,12 @@ public class CliFrontendRunTest extends CliFrontendTestBase {
// test configure savepoint path (no ignore flag)
{
String[] parameters = {"-s", "expectedSavepointPath", getTestJarPath()};
- RunOptions options = CliFrontendParser.parseRunCommand(parameters);
- SavepointRestoreSettings savepointSettings = options.getSavepointRestoreSettings();
+
+ CommandLine commandLine = CliFrontendParser.parse(CliFrontendParser.RUN_OPTIONS, parameters, true);
+ ProgramOptions programOptions = new ProgramOptions(commandLine);
+ ExecutionConfigAccessor executionOptions = ExecutionConfigAccessor.fromProgramOptions(programOptions);
+
+ SavepointRestoreSettings savepointSettings = executionOptions.getSavepointRestoreSettings();
assertTrue(savepointSettings.restoreSavepoint());
assertEquals("expectedSavepointPath", savepointSettings.getRestorePath());
assertFalse(savepointSettings.allowNonRestoredState());
@@ -91,8 +96,12 @@ public class CliFrontendRunTest extends CliFrontendTestBase {
// test configure savepoint path (with ignore flag)
{
String[] parameters = {"-s", "expectedSavepointPath", "-n", getTestJarPath()};
- RunOptions options = CliFrontendParser.parseRunCommand(parameters);
- SavepointRestoreSettings savepointSettings = options.getSavepointRestoreSettings();
+
+ CommandLine commandLine = CliFrontendParser.parse(CliFrontendParser.RUN_OPTIONS, parameters, true);
+ ProgramOptions programOptions = new ProgramOptions(commandLine);
+ ExecutionConfigAccessor executionOptions = ExecutionConfigAccessor.fromProgramOptions(programOptions);
+
+ SavepointRestoreSettings savepointSettings = executionOptions.getSavepointRestoreSettings();
assertTrue(savepointSettings.restoreSavepoint());
assertEquals("expectedSavepointPath", savepointSettings.getRestorePath());
assertTrue(savepointSettings.allowNonRestoredState());
@@ -102,12 +111,15 @@ public class CliFrontendRunTest extends CliFrontendTestBase {
{
String[] parameters =
{ getTestJarPath(), "-arg1", "value1", "justavalue", "--arg2", "value2"};
- RunOptions options = CliFrontendParser.parseRunCommand(parameters);
- assertEquals("-arg1", options.getProgramArgs()[0]);
- assertEquals("value1", options.getProgramArgs()[1]);
- assertEquals("justavalue", options.getProgramArgs()[2]);
- assertEquals("--arg2", options.getProgramArgs()[3]);
- assertEquals("value2", options.getProgramArgs()[4]);
+
+ CommandLine commandLine = CliFrontendParser.parse(CliFrontendParser.RUN_OPTIONS, parameters, true);
+ ProgramOptions programOptions = new ProgramOptions(commandLine);
+
+ assertEquals("-arg1", programOptions.getProgramArgs()[0]);
+ assertEquals("value1", programOptions.getProgramArgs()[1]);
+ assertEquals("justavalue", programOptions.getProgramArgs()[2]);
+ assertEquals("--arg2", programOptions.getProgramArgs()[3]);
+ assertEquals("value2", programOptions.getProgramArgs()[4]);
}
}
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/DeploymentOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/DeploymentOptions.java
index a032205..a09fd68 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/DeploymentOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/DeploymentOptions.java
@@ -37,4 +37,10 @@ public class DeploymentOptions {
key("execution.attached")
.defaultValue(false)
.withDescription("Specifies if the pipeline is submitted in attached or detached mode.");
+
+ public static final ConfigOption<Boolean> SHUTDOWN_IF_ATTACHED =
+ key("execution.shutdown-on-attached-exit")
+ .defaultValue(false)
+ .withDescription("If the job is submitted in attached mode, perform a best-effort cluster shutdown " +
+ "when the CLI is terminated abruptly, e.g., in response to a user interrupt, such as typing Ctrl + C.");
}
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/DeploymentOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/PipelineOptions.java
similarity index 51%
copy from flink-core/src/main/java/org/apache/flink/configuration/DeploymentOptions.java
copy to flink-core/src/main/java/org/apache/flink/configuration/PipelineOptions.java
index a032205..6ae8a31 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/DeploymentOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/PipelineOptions.java
@@ -23,18 +23,27 @@ import org.apache.flink.annotation.PublicEvolving;
import static org.apache.flink.configuration.ConfigOptions.key;
/**
- * The {@link ConfigOption configuration options} relevant for all Executors.
+ * The {@link ConfigOption configuration options} for job execution.
*/
@PublicEvolving
-public class DeploymentOptions {
+public class PipelineOptions {
- public static final ConfigOption<String> TARGET =
- key("execution.target")
+ public static final String LIST_SEPARATOR = ";";
+
+ /**
+ * A list of jar files that contain the user-defined function (UDF) classes and all classes used from within the UDFs.
+ */
+ public static final ConfigOption<String> JARS =
+ key("pipeline.jars")
.noDefaultValue()
- .withDescription("The deployment target for the execution, e.g. \"local\" for local execution.");
+ .withDescription("A semicolon-separated list of the jars to package with the job jars to be sent to the cluster. These have to be valid paths.");
- public static final ConfigOption<Boolean> ATTACHED =
- key("execution.attached")
- .defaultValue(false)
- .withDescription("Specifies if the pipeline is submitted in attached or detached mode.");
+ /**
+ * A list of URLs that are added to the classpath of each user code classloader of the program.
+ * Paths must specify a protocol (e.g. file://) and be accessible on all nodes
+ */
+ public static final ConfigOption<String> CLASSPATHS =
+ key("pipeline.classpaths")
+ .noDefaultValue()
+ .withDescription("A semicolon-separated list of the classpaths to package with the job jars to be sent to the cluster. These have to be valid URLs.");
}
diff --git a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
index 999f85a..deb4e03 100644
--- a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
+++ b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java
@@ -58,6 +58,7 @@ public class ConfigOptionsDocGenerator {
static final OptionsClassLocation[] LOCATIONS = new OptionsClassLocation[]{
new OptionsClassLocation("flink-core", "org.apache.flink.configuration"),
new OptionsClassLocation("flink-runtime", "org.apache.flink.runtime.shuffle"),
+ new OptionsClassLocation("flink-runtime", "org.apache.flink.runtime.jobgraph"),
new OptionsClassLocation("flink-yarn", "org.apache.flink.yarn.configuration"),
new OptionsClassLocation("flink-mesos", "org.apache.flink.mesos.configuration"),
new OptionsClassLocation("flink-mesos", "org.apache.flink.mesos.runtime.clusterframework"),
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/DeploymentOptions.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/SavepointConfigOptions.java
similarity index 50%
copy from flink-core/src/main/java/org/apache/flink/configuration/DeploymentOptions.java
copy to flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/SavepointConfigOptions.java
index a032205..86fd389 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/DeploymentOptions.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/SavepointConfigOptions.java
@@ -16,25 +16,34 @@
* limitations under the License.
*/
-package org.apache.flink.configuration;
+package org.apache.flink.runtime.jobgraph;
import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.configuration.ConfigOption;
import static org.apache.flink.configuration.ConfigOptions.key;
/**
- * The {@link ConfigOption configuration options} relevant for all Executors.
+ * The {@link ConfigOption configuration options} used when restoring from a savepoint.
*/
@PublicEvolving
-public class DeploymentOptions {
+public class SavepointConfigOptions {
- public static final ConfigOption<String> TARGET =
- key("execution.target")
+ /**
+ * The path to a savepoint that will be used to bootstrap the pipeline's state.
+ */
+ public static final ConfigOption<String> SAVEPOINT_PATH =
+ key("execution.savepoint.path")
.noDefaultValue()
- .withDescription("The deployment target for the execution, e.g. \"local\" for local execution.");
+ .withDescription("Path to a savepoint to restore the job from (for example hdfs:///flink/savepoint-1537).");
- public static final ConfigOption<Boolean> ATTACHED =
- key("execution.attached")
+ /**
+ * A flag indicating if we allow Flink to skip savepoint state that cannot be restored,
+ * e.g. because the corresponding operator has been removed.
+ */
+ public static final ConfigOption<Boolean> SAVEPOINT_IGNORE_UNCLAIMED_STATE =
+ key("execution.savepoint.ignore-unclaimed-state")
.defaultValue(false)
- .withDescription("Specifies if the pipeline is submitted in attached or detached mode.");
+ .withDescription("Allow to skip savepoint state that cannot be restored. " +
+ "Allow this if you removed an operator from your pipeline after the savepoint was triggered.");
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/SavepointRestoreSettings.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/SavepointRestoreSettings.java
index 953d11c..b7fee4e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/SavepointRestoreSettings.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/SavepointRestoreSettings.java
@@ -18,6 +18,8 @@
package org.apache.flink.runtime.jobgraph;
+import org.apache.flink.configuration.Configuration;
+
import java.io.Serializable;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -131,4 +133,19 @@ public class SavepointRestoreSettings implements Serializable {
return new SavepointRestoreSettings(savepointPath, allowNonRestoredState);
}
+ // -------------------------- Parsing to and from a configuration object ------------------------------------
+
+ public static void toConfiguration(final SavepointRestoreSettings savepointRestoreSettings, final Configuration configuration) {
+ configuration.setBoolean(SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE, savepointRestoreSettings.allowNonRestoredState());
+ final String savepointPath = savepointRestoreSettings.getRestorePath();
+ if (savepointPath != null) {
+ configuration.setString(SavepointConfigOptions.SAVEPOINT_PATH, savepointPath);
+ }
+ }
+
+ public static SavepointRestoreSettings fromConfiguration(final Configuration configuration) {
+ final String savepointPath = configuration.getString(SavepointConfigOptions.SAVEPOINT_PATH);
+ final boolean allowNonRestored = configuration.getBoolean(SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE);
+ return savepointPath == null ? SavepointRestoreSettings.none() : SavepointRestoreSettings.forPath(savepointPath, allowNonRestored);
+ }
}
diff --git a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java
index 580b0be..287c9da 100644
--- a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java
+++ b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java
@@ -27,7 +27,8 @@ import org.apache.flink.client.ClientUtils;
import org.apache.flink.client.FlinkPipelineTranslationUtil;
import org.apache.flink.client.cli.CliArgsException;
import org.apache.flink.client.cli.CustomCommandLine;
-import org.apache.flink.client.cli.RunOptions;
+import org.apache.flink.client.cli.ExecutionConfigAccessor;
+import org.apache.flink.client.cli.ProgramOptions;
import org.apache.flink.client.deployment.ClusterClientFactory;
import org.apache.flink.client.deployment.ClusterClientServiceLoader;
import org.apache.flink.client.deployment.ClusterDescriptor;
@@ -118,7 +119,7 @@ public class ExecutionContext<T> {
private final Configuration flinkConfig;
private final Configuration executorConfig;
private final ClusterClientFactory<T> clusterClientFactory;
- private final RunOptions runOptions;
+ private final ExecutionConfigAccessor executionParameters;
private final T clusterId;
private final ClusterSpecification clusterSpec;
@@ -173,7 +174,7 @@ public class ExecutionContext<T> {
clusterClientFactory = serviceLoader.getClusterClientFactory(executorConfig);
checkState(clusterClientFactory != null);
- runOptions = createRunOptions(commandLine);
+ executionParameters = createExecutionParameterProvider(commandLine);
clusterId = clusterClientFactory.getClusterId(executorConfig);
clusterSpec = clusterClientFactory.getClusterSpecification(executorConfig);
}
@@ -251,9 +252,10 @@ public class ExecutionContext<T> {
throw new SqlExecutionException("Could not find a matching deployment.");
}
- private static RunOptions createRunOptions(CommandLine commandLine) {
+ private static ExecutionConfigAccessor createExecutionParameterProvider(CommandLine commandLine) {
try {
- return new RunOptions(commandLine);
+ final ProgramOptions programOptions = new ProgramOptions(commandLine);
+ return ExecutionConfigAccessor.fromProgramOptions(programOptions);
} catch (CliArgsException e) {
throw new SqlExecutionException("Invalid deployment run options.", e);
}
@@ -462,8 +464,8 @@ public class ExecutionContext<T> {
parallelism);
ClientUtils.addJarFiles(jobGraph, dependencies);
- jobGraph.setClasspaths(runOptions.getClasspaths());
- jobGraph.setSavepointRestoreSettings(runOptions.getSavepointRestoreSettings());
+ jobGraph.setClasspaths(executionParameters.getClasspaths());
+ jobGraph.setSavepointRestoreSettings(executionParameters.getSavepointRestoreSettings());
return jobGraph;
}