You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by di...@apache.org on 2022/03/25 09:33:23 UTC
[flink] branch release-1.15 updated: [FLINK-26847][python] Ensure command line option '-py' works in YARN application mode
This is an automated email from the ASF dual-hosted git repository.
dianfu pushed a commit to branch release-1.15
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.15 by this push:
new 608f5e1 [FLINK-26847][python] Ensure command line option '-py' works in YARN application mode
608f5e1 is described below
commit 608f5e1761b4fc011aec6cc958ef375d6992814c
Author: Dian Fu <di...@apache.org>
AuthorDate: Thu Mar 24 21:04:05 2022 +0800
[FLINK-26847][python] Ensure command line option '-py' works in YARN application mode
This closes #19227.
---
.../java/org/apache/flink/client/python/PythonDriver.java | 8 ++++++--
.../flink/client/python/PythonDriverOptionsParserFactory.java | 6 ++++++
.../java/org/apache/flink/client/python/PythonDriverTest.java | 11 ++++++-----
3 files changed, 18 insertions(+), 7 deletions(-)
diff --git a/flink-python/src/main/java/org/apache/flink/client/python/PythonDriver.java b/flink-python/src/main/java/org/apache/flink/client/python/PythonDriver.java
index 57b1eb2..a25ee91 100644
--- a/flink-python/src/main/java/org/apache/flink/client/python/PythonDriver.java
+++ b/flink-python/src/main/java/org/apache/flink/client/python/PythonDriver.java
@@ -154,10 +154,14 @@ public final class PythonDriver {
*/
static List<String> constructPythonCommands(final PythonDriverOptions pythonDriverOptions) {
final List<String> commands = new ArrayList<>();
+ commands.add("-m");
if (pythonDriverOptions.getEntryPointScript().isPresent()) {
- commands.add(pythonDriverOptions.getEntryPointScript().get());
+ String pythonFileName = pythonDriverOptions.getEntryPointScript().get();
+ commands.add(
+ pythonFileName.substring(
+ pythonFileName.lastIndexOf(File.separator) + 1,
+ pythonFileName.lastIndexOf(".py")));
} else {
- commands.add("-m");
commands.add(pythonDriverOptions.getEntryPointModule());
}
commands.addAll(pythonDriverOptions.getProgramArgs());
diff --git a/flink-python/src/main/java/org/apache/flink/client/python/PythonDriverOptionsParserFactory.java b/flink-python/src/main/java/org/apache/flink/client/python/PythonDriverOptionsParserFactory.java
index 6e5663e..ded5577 100644
--- a/flink-python/src/main/java/org/apache/flink/client/python/PythonDriverOptionsParserFactory.java
+++ b/flink-python/src/main/java/org/apache/flink/client/python/PythonDriverOptionsParserFactory.java
@@ -54,6 +54,12 @@ final class PythonDriverOptionsParserFactory implements ParserResultFactory<Pyth
throw new FlinkParseException("Cannot use options -py and -pym simultaneously.");
} else if (commandLine.hasOption(PY_OPTION.getOpt())) {
entryPointScript = commandLine.getOptionValue(PY_OPTION.getOpt());
+ if (!entryPointScript.endsWith(".py")) {
+ throw new FlinkParseException(
+ String.format(
+ "It only accepts Python file which ends with '.py' for option '-py', got '%s'.",
+ entryPointScript));
+ }
} else if (commandLine.hasOption(PYMODULE_OPTION.getOpt())) {
entryPointModule = commandLine.getOptionValue(PYMODULE_OPTION.getOpt());
} else {
diff --git a/flink-python/src/test/java/org/apache/flink/client/python/PythonDriverTest.java b/flink-python/src/test/java/org/apache/flink/client/python/PythonDriverTest.java
index 773353d..fe46868 100644
--- a/flink-python/src/test/java/org/apache/flink/client/python/PythonDriverTest.java
+++ b/flink-python/src/test/java/org/apache/flink/client/python/PythonDriverTest.java
@@ -65,11 +65,12 @@ public class PythonDriverTest {
args.add("--input");
args.add("in.txt");
- PythonDriverOptions pythonDriverOptions = new PythonDriverOptions(null, "xxx", args);
+ PythonDriverOptions pythonDriverOptions = new PythonDriverOptions(null, "xxx.py", args);
List<String> commands = PythonDriver.constructPythonCommands(pythonDriverOptions);
- Assert.assertEquals(3, commands.size());
- Assert.assertEquals(commands.get(0), "xxx");
- Assert.assertEquals(commands.get(1), "--input");
- Assert.assertEquals(commands.get(2), "in.txt");
+ Assert.assertEquals(4, commands.size());
+ Assert.assertEquals(commands.get(0), "-m");
+ Assert.assertEquals(commands.get(1), "xxx");
+ Assert.assertEquals(commands.get(2), "--input");
+ Assert.assertEquals(commands.get(3), "in.txt");
}
}