You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by go...@apache.org on 2020/11/27 01:59:02 UTC
[flink] branch release-1.12 updated (84b6631 -> 1aa1b8c)
This is an automated email from the ASF dual-hosted git repository.
godfrey pushed a change to branch release-1.12
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 84b6631 [FLINK-20351] Correct the logging of the location of a task failure on the JM
new 2c8e9d4 [FLINK-18545][configuration] Introduce `pipeline.name` to allow users to specify job name by configuration
new beaeead [FLINK-18545][configuration] Allow users to specify job name by configuration in ExecutionEnvironment
new 5b73483 [FLINK-18545][python] Specify job name by `pipeline.name` for flink-python
new 1aa1b8c [FLINK-18545][table] Specify job name by `pipeline.name` for sql job
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../generated/pipeline_configuration.html | 6 ++++
.../flink/client/ExecutionEnvironmentTest.java | 35 ++++++++++++++++++++++
.../flink/configuration/PipelineOptions.java | 10 +++++++
.../flink/api/java/ExecutionEnvironment.java | 19 +++++++-----
.../apache/flink/python/util/PythonConfigUtil.java | 4 ++-
.../flink/python/util/PythonConfigUtilTest.java | 17 +++++++++++
.../environment/StreamExecutionEnvironment.java | 15 +++++++---
.../api/StreamExecutionEnvironmentTest.java | 33 ++++++++++++++++++++
.../table/api/internal/TableEnvironmentImpl.java | 10 +++++--
9 files changed, 134 insertions(+), 15 deletions(-)
[flink] 04/04: [FLINK-18545][table] Specify job name by
`pipeline.name` for sql job
Posted by go...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
godfrey pushed a commit to branch release-1.12
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 1aa1b8c6e6d4628b01ff5503f3a680152680caeb
Author: godfreyhe <go...@163.com>
AuthorDate: Tue Nov 17 23:32:07 2020 +0800
[FLINK-18545][table] Specify job name by `pipeline.name` for sql job
(cherry picked from commit 88d0ee002c38f8a17895d4485703787a04f67c50)
---
.../apache/flink/table/api/internal/TableEnvironmentImpl.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java
index 8ab6974..a6a3c51 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java
@@ -23,6 +23,7 @@ import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.dag.Pipeline;
import org.apache.flink.api.dag.Transformation;
+import org.apache.flink.configuration.PipelineOptions;
import org.apache.flink.core.execution.JobClient;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.api.EnvironmentSettings;
@@ -673,7 +674,7 @@ public class TableEnvironmentImpl implements TableEnvironmentInternal {
public TableResult executeInternal(List<ModifyOperation> operations) {
List<Transformation<?>> transformations = translate(operations);
List<String> sinkIdentifierNames = extractSinkIdentifierNames(operations);
- String jobName = "insert-into_" + String.join(",", sinkIdentifierNames);
+ String jobName = getJobName("insert-into_" + String.join(",", sinkIdentifierNames));
Pipeline pipeline = execEnv.createPipeline(transformations, tableConfig, jobName);
try {
JobClient jobClient = execEnv.executeAsync(pipeline);
@@ -700,7 +701,8 @@ public class TableEnvironmentImpl implements TableEnvironmentInternal {
public TableResult executeInternal(QueryOperation operation) {
SelectSinkOperation sinkOperation = new SelectSinkOperation(operation);
List<Transformation<?>> transformations = translate(Collections.singletonList(sinkOperation));
- Pipeline pipeline = execEnv.createPipeline(transformations, tableConfig, "collect");
+ String jobName = getJobName("collect");
+ Pipeline pipeline = execEnv.createPipeline(transformations, tableConfig, jobName);
try {
JobClient jobClient = execEnv.executeAsync(pipeline);
SelectResultProvider resultProvider = sinkOperation.getSelectResultProvider();
@@ -1170,6 +1172,10 @@ public class TableEnvironmentImpl implements TableEnvironmentInternal {
).collect(Collectors.toList());
}
+ private String getJobName(String defaultJobName) {
+ return tableConfig.getConfiguration().getString(PipelineOptions.NAME, defaultJobName);
+ }
+
/** Get catalog from catalogName or throw a ValidationException if the catalog not exists. */
private Catalog getCatalogOrThrowException(String catalogName) {
return getCatalog(catalogName)
[flink] 03/04: [FLINK-18545][python] Specify job name by
`pipeline.name` for flink-python
Posted by go...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
godfrey pushed a commit to branch release-1.12
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 5b73483bf091894c668378a58df9cfd6a584796e
Author: godfreyhe <go...@163.com>
AuthorDate: Tue Nov 17 23:04:37 2020 +0800
[FLINK-18545][python] Specify job name by `pipeline.name` for flink-python
(cherry picked from commit 56edecd09104b7f876bf2ca4f9e6c4a590d0253b)
---
.../org/apache/flink/python/util/PythonConfigUtil.java | 4 +++-
.../apache/flink/python/util/PythonConfigUtilTest.java | 17 +++++++++++++++++
2 files changed, 20 insertions(+), 1 deletion(-)
diff --git a/flink-python/src/main/java/org/apache/flink/python/util/PythonConfigUtil.java b/flink-python/src/main/java/org/apache/flink/python/util/PythonConfigUtil.java
index 18e5371..bf323ab 100644
--- a/flink-python/src/main/java/org/apache/flink/python/util/PythonConfigUtil.java
+++ b/flink-python/src/main/java/org/apache/flink/python/util/PythonConfigUtil.java
@@ -22,6 +22,7 @@ import org.apache.flink.api.connector.source.Boundedness;
import org.apache.flink.api.dag.Transformation;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.ExecutionOptions;
+import org.apache.flink.configuration.PipelineOptions;
import org.apache.flink.core.memory.ManagedMemoryUseCase;
import org.apache.flink.python.PythonConfig;
import org.apache.flink.python.PythonOptions;
@@ -153,7 +154,8 @@ public class PythonConfigUtil {
}
}
- StreamGraph streamGraph = env.getStreamGraph(StreamExecutionEnvironment.DEFAULT_JOB_NAME, clearTransformations);
+ String jobName = getEnvironmentConfig(env).getString(PipelineOptions.NAME, StreamExecutionEnvironment.DEFAULT_JOB_NAME);
+ StreamGraph streamGraph = env.getStreamGraph(jobName, clearTransformations);
Collection<StreamNode> streamNodes = streamGraph.getStreamNodes();
for (StreamNode streamNode : streamNodes) {
alignStreamNode(streamNode, streamGraph);
diff --git a/flink-python/src/test/java/org/apache/flink/python/util/PythonConfigUtilTest.java b/flink-python/src/test/java/org/apache/flink/python/util/PythonConfigUtilTest.java
index a6c9863..4f1da09 100644
--- a/flink-python/src/test/java/org/apache/flink/python/util/PythonConfigUtilTest.java
+++ b/flink-python/src/test/java/org/apache/flink/python/util/PythonConfigUtilTest.java
@@ -18,12 +18,17 @@
package org.apache.flink.python.util;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.PipelineOptions;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
+import org.apache.flink.streaming.api.graph.StreamGraph;
import org.junit.Test;
import java.lang.reflect.InvocationTargetException;
+import java.util.Collections;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
/**
@@ -37,4 +42,16 @@ public class PythonConfigUtilTest {
Configuration envConfig = PythonConfigUtil.getEnvConfigWithDependencies(executionEnvironment);
assertNotNull(envConfig);
}
+
+ @Test
+ public void testJobName() throws IllegalAccessException, NoSuchMethodException, InvocationTargetException, NoSuchFieldException {
+ String jobName = "MyTestJob";
+ Configuration config = new Configuration();
+ config.set(PipelineOptions.NAME, jobName);
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(config);
+
+ env.fromCollection(Collections.singletonList("test")).addSink(new DiscardingSink<>());
+ StreamGraph streamGraph = PythonConfigUtil.generateStreamGraphWithDependencies(env, true);
+ assertEquals(jobName, streamGraph.getJobName());
+ }
}
[flink] 02/04: [FLINK-18545][configuration] Allow users to specify
job name by configuration in ExecutionEnvironment
Posted by go...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
godfrey pushed a commit to branch release-1.12
in repository https://gitbox.apache.org/repos/asf/flink.git
commit beaeead0a582fa646b6f26bcafcc106d4e07a293
Author: godfreyhe <go...@163.com>
AuthorDate: Thu Nov 26 11:39:46 2020 +0800
[FLINK-18545][configuration] Allow users to specify job name by configuration in ExecutionEnvironment
(cherry picked from commit eaad1678f2de23addcee7592e982f61d690777a7)
---
.../flink/client/ExecutionEnvironmentTest.java | 35 ++++++++++++++++++++++
.../flink/api/java/ExecutionEnvironment.java | 19 +++++++-----
2 files changed, 46 insertions(+), 8 deletions(-)
diff --git a/flink-clients/src/test/java/org/apache/flink/client/ExecutionEnvironmentTest.java b/flink-clients/src/test/java/org/apache/flink/client/ExecutionEnvironmentTest.java
index f512fe8..5f22fbd 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/ExecutionEnvironmentTest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/ExecutionEnvironmentTest.java
@@ -19,10 +19,13 @@
package org.apache.flink.client;
+import org.apache.flink.api.common.Plan;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.PipelineOptions;
import org.apache.flink.util.TestLogger;
import org.junit.Test;
@@ -30,6 +33,7 @@ import org.junit.Test;
import java.io.Serializable;
import static junit.framework.TestCase.fail;
+import static org.junit.Assert.assertTrue;
/**
@@ -63,4 +67,35 @@ public class ExecutionEnvironmentTest extends TestLogger implements Serializable
fail("Consecutive #getExecutionPlan calls caused an exception.");
}
}
+
+ @Test
+ public void testDefaultJobName() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ testJobName("Flink Java Job at", env);
+ }
+
+ @Test
+ public void testUserDefinedJobName() {
+ String jobName = "MyTestJob";
+ Configuration config = new Configuration();
+ config.set(PipelineOptions.NAME, jobName);
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(config);
+ testJobName(jobName, env);
+ }
+
+ @Test
+ public void testUserDefinedJobNameWithConfigure() {
+ String jobName = "MyTestJob";
+ Configuration config = new Configuration();
+ config.set(PipelineOptions.NAME, jobName);
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.configure(config, this.getClass().getClassLoader());
+ testJobName(jobName, env);
+ }
+
+ private void testJobName(String prefixOfExpectedJobName, ExecutionEnvironment env) {
+ env.fromElements(1, 2, 3).writeAsText("/dev/null");
+ Plan plan = env.createProgramPlan();
+ assertTrue(plan.getJobName().startsWith(prefixOfExpectedJobName));
+ }
}
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
index 52a0d85..c3f3268 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
@@ -418,6 +418,8 @@ public class ExecutionEnvironment {
this.cacheFile.clear();
this.cacheFile.addAll(DistributedCache.parseCachedFilesFromString(f));
});
+ configuration.getOptional(PipelineOptions.NAME)
+ .ifPresent(jobName -> this.getConfiguration().set(PipelineOptions.NAME, jobName));
config.configure(configuration, classLoader);
}
@@ -870,7 +872,7 @@ public class ExecutionEnvironment {
* @throws Exception Thrown, if the program executions fails.
*/
public JobExecutionResult execute() throws Exception {
- return execute(getDefaultName());
+ return execute(getJobName());
}
/**
@@ -945,7 +947,7 @@ public class ExecutionEnvironment {
*/
@PublicEvolving
public final JobClient executeAsync() throws Exception {
- return executeAsync(getDefaultName());
+ return executeAsync(getJobName());
}
/**
@@ -998,7 +1000,7 @@ public class ExecutionEnvironment {
* @throws Exception Thrown, if the compiler could not be instantiated.
*/
public String getExecutionPlan() throws Exception {
- Plan p = createProgramPlan(getDefaultName(), false);
+ Plan p = createProgramPlan(getJobName(), false);
return ExecutionPlanUtil.getExecutionPlanAsJSON(p);
}
@@ -1051,7 +1053,7 @@ public class ExecutionEnvironment {
*/
@Internal
public Plan createProgramPlan() {
- return createProgramPlan(getDefaultName());
+ return createProgramPlan(getJobName());
}
/**
@@ -1122,12 +1124,13 @@ public class ExecutionEnvironment {
}
/**
- * Gets a default job name, based on the timestamp when this method is invoked.
+ * Gets the job name. If user defined job name is not found in the configuration,
+ * the default name based on the timestamp when this method is invoked will return.
*
- * @return A default job name.
+ * @return A job name.
*/
- private static String getDefaultName() {
- return "Flink Java Job at " + Calendar.getInstance().getTime();
+ private String getJobName() {
+ return configuration.getString(PipelineOptions.NAME, "Flink Java Job at " + Calendar.getInstance().getTime());
}
// --------------------------------------------------------------------------------------------
[flink] 01/04: [FLINK-18545][configuration] Introduce
`pipeline.name` to allow users to specify job name by configuration
Posted by go...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
godfrey pushed a commit to branch release-1.12
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 2c8e9d42cea40fd01be5cbdfdb791a4ecb5cdd07
Author: godfreyhe <go...@163.com>
AuthorDate: Tue Nov 17 22:50:50 2020 +0800
[FLINK-18545][configuration] Introduce `pipeline.name` to allow users to specify job name by configuration
(cherry picked from commit 78d6ef6f41b24977c531cfcb430cf184edc6f5ed)
---
.../generated/pipeline_configuration.html | 6 ++++
.../flink/configuration/PipelineOptions.java | 10 +++++++
.../environment/StreamExecutionEnvironment.java | 15 +++++++---
.../api/StreamExecutionEnvironmentTest.java | 33 ++++++++++++++++++++++
4 files changed, 60 insertions(+), 4 deletions(-)
diff --git a/docs/_includes/generated/pipeline_configuration.html b/docs/_includes/generated/pipeline_configuration.html
index f3fc216..a7f970c 100644
--- a/docs/_includes/generated/pipeline_configuration.html
+++ b/docs/_includes/generated/pipeline_configuration.html
@@ -87,6 +87,12 @@
<td>The program-wide maximum parallelism used for operators which haven't specified a maximum parallelism. The maximum parallelism specifies the upper limit for dynamic scaling and the number of key groups used for partitioned state.</td>
</tr>
<tr>
+ <td><h5>pipeline.name</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The job name used for printing and logging.</td>
+ </tr>
+ <tr>
<td><h5>pipeline.object-reuse</h5></td>
<td style="word-wrap: break-word;">false</td>
<td>Boolean</td>
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/PipelineOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/PipelineOptions.java
index 3c405b3..a687cba 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/PipelineOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/PipelineOptions.java
@@ -36,6 +36,16 @@ import static org.apache.flink.configuration.description.TextElement.text;
*/
@PublicEvolving
public class PipelineOptions {
+
+ /**
+ * The job name used for printing and logging.
+ */
+ public static final ConfigOption<String> NAME =
+ key("pipeline.name")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("The job name used for printing and logging.");
+
/**
* A list of jar files that contain the user-defined function (UDF) classes and all classes used from within the UDFs.
*/
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 9effe65..f458c27 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -839,6 +839,9 @@ public class StreamExecutionEnvironment {
configuration.getOptional(ExecutionOptions.USE_BATCH_STATE_BACKEND).ifPresent(
sortInputs -> this.getConfiguration().set(ExecutionOptions.USE_BATCH_STATE_BACKEND, sortInputs)
);
+ configuration.getOptional(PipelineOptions.NAME).ifPresent(
+ jobName -> this.getConfiguration().set(PipelineOptions.NAME, jobName)
+ );
config.configure(configuration, classLoader);
checkpointCfg.configure(configuration);
}
@@ -1798,7 +1801,7 @@ public class StreamExecutionEnvironment {
* @throws Exception which occurs during job execution.
*/
public JobExecutionResult execute() throws Exception {
- return execute(DEFAULT_JOB_NAME);
+ return execute(getJobName());
}
/**
@@ -1891,7 +1894,7 @@ public class StreamExecutionEnvironment {
*/
@PublicEvolving
public final JobClient executeAsync() throws Exception {
- return executeAsync(DEFAULT_JOB_NAME);
+ return executeAsync(getJobName());
}
/**
@@ -1958,7 +1961,7 @@ public class StreamExecutionEnvironment {
*/
@Internal
public StreamGraph getStreamGraph() {
- return getStreamGraph(DEFAULT_JOB_NAME);
+ return getStreamGraph(getJobName());
}
/**
@@ -2018,7 +2021,7 @@ public class StreamExecutionEnvironment {
* @return The execution plan of the program, as a JSON String.
*/
public String getExecutionPlan() {
- return getStreamGraph(DEFAULT_JOB_NAME, false).getStreamingPlanAsJSON();
+ return getStreamGraph(getJobName(), false).getStreamingPlanAsJSON();
}
/**
@@ -2350,4 +2353,8 @@ public class StreamExecutionEnvironment {
}
return (T) resolvedTypeInfo;
}
+
+ private String getJobName() {
+ return configuration.getString(PipelineOptions.NAME, DEFAULT_JOB_NAME);
+ }
}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java
index d0bc4b0..e8d5add 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java
@@ -23,6 +23,8 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.java.typeutils.GenericTypeInfo;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.PipelineOptions;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
@@ -270,6 +272,37 @@ public class StreamExecutionEnvironmentTest {
}
@Test
+ public void testDefaultJobName() {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ testJobName(StreamExecutionEnvironment.DEFAULT_JOB_NAME, env);
+ }
+
+ @Test
+ public void testUserDefinedJobName() {
+ String jobName = "MyTestJob";
+ Configuration config = new Configuration();
+ config.set(PipelineOptions.NAME, jobName);
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(config);
+ testJobName(jobName, env);
+ }
+
+ @Test
+ public void testUserDefinedJobNameWithConfigure() {
+ String jobName = "MyTestJob";
+ Configuration config = new Configuration();
+ config.set(PipelineOptions.NAME, jobName);
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.configure(config, this.getClass().getClassLoader());
+ testJobName(jobName, env);
+ }
+
+ private void testJobName(String expectedJobName, StreamExecutionEnvironment env) {
+ env.fromElements(1, 2, 3).print();
+ StreamGraph streamGraph = env.getStreamGraph();
+ assertEquals(expectedJobName, streamGraph.getJobName());
+ }
+
+ @Test
public void testAddSourceWithUserDefinedTypeInfo() {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStreamSource<Row> source1 = env.addSource(new RowSourceFunction(), Types.ROW(Types.STRING));