You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by zh...@apache.org on 2024/10/10 15:17:27 UTC
(flink) branch master updated: [FLINK-36445][runtime] Introduce ExecutionPlan interface.
This is an automated email from the ASF dual-hosted git repository.
zhuzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 417f1427cd0 [FLINK-36445][runtime] Introduce ExecutionPlan interface.
417f1427cd0 is described below
commit 417f1427cd04f77384918e82a393720abcac6d6d
Author: JunRuiLee <jr...@gmail.com>
AuthorDate: Wed Oct 9 15:49:06 2024 +0800
[FLINK-36445][runtime] Introduce ExecutionPlan interface.
This closes #25469.
---
.../flink/runtime/dispatcher/Dispatcher.java | 22 +--
.../JobMasterServiceLeadershipRunnerFactory.java | 2 +-
.../apache/flink/runtime/jobgraph/JobGraph.java | 75 +++++---
.../flink/streaming/api/graph/ExecutionPlan.java | 202 +++++++++++++++++++++
4 files changed, 254 insertions(+), 47 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
index 10b8ad11794..32ebf210fe1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
@@ -22,7 +22,6 @@ import org.apache.flink.annotation.Internal;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobStatus;
-import org.apache.flink.api.common.operators.ResourceSpec;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.BlobServerOptions;
import org.apache.flink.configuration.CheckpointingOptions;
@@ -541,7 +540,7 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId>
// job with the given jobID is not terminated, yet
return FutureUtils.completedExceptionally(
DuplicateJobSubmissionException.of(jobID));
- } else if (isPartialResourceConfigured(jobGraph)) {
+ } else if (jobGraph.isPartialResourceConfigured()) {
return FutureUtils.completedExceptionally(
new JobSubmissionException(
jobID,
@@ -583,25 +582,6 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId>
return jobResultStore.hasJobResultEntryAsync(jobId);
}
- private boolean isPartialResourceConfigured(JobGraph jobGraph) {
- boolean hasVerticesWithUnknownResource = false;
- boolean hasVerticesWithConfiguredResource = false;
-
- for (JobVertex jobVertex : jobGraph.getVertices()) {
- if (jobVertex.getMinResources() == ResourceSpec.UNKNOWN) {
- hasVerticesWithUnknownResource = true;
- } else {
- hasVerticesWithConfiguredResource = true;
- }
-
- if (hasVerticesWithUnknownResource && hasVerticesWithConfiguredResource) {
- return true;
- }
- }
-
- return false;
- }
-
private CompletableFuture<Acknowledge> internalSubmitJob(JobGraph jobGraph) {
applyParallelismOverrides(jobGraph);
log.info("Submitting job '{}' ({}).", jobGraph.getName(), jobGraph.getJobID());
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/JobMasterServiceLeadershipRunnerFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/JobMasterServiceLeadershipRunnerFactory.java
index 718db104ed9..98500425608 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/JobMasterServiceLeadershipRunnerFactory.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/JobMasterServiceLeadershipRunnerFactory.java
@@ -64,7 +64,7 @@ public enum JobMasterServiceLeadershipRunnerFactory implements JobManagerRunnerF
long initializationTimestamp)
throws Exception {
- checkArgument(jobGraph.getNumberOfVertices() > 0, "The given job is empty");
+ checkArgument(!jobGraph.isEmpty(), "The given job is empty");
final JobMasterConfiguration jobMasterConfiguration =
JobMasterConfiguration.fromConfiguration(configuration);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java
index 99c2fb75bd7..4d57bcd1d4f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java
@@ -22,8 +22,7 @@ import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.cache.DistributedCache;
-import org.apache.flink.configuration.ConfigOption;
-import org.apache.flink.configuration.ConfigOptions;
+import org.apache.flink.api.common.operators.ResourceSpec;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.execution.JobStatusHook;
import org.apache.flink.core.fs.Path;
@@ -31,6 +30,7 @@ import org.apache.flink.runtime.blob.PermanentBlobKey;
import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
+import org.apache.flink.streaming.api.graph.ExecutionPlan;
import org.apache.flink.util.InstantiationUtil;
import org.apache.flink.util.IterableUtils;
import org.apache.flink.util.SerializedValue;
@@ -38,7 +38,6 @@ import org.apache.flink.util.SerializedValue;
import javax.annotation.Nullable;
import java.io.IOException;
-import java.io.Serializable;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
@@ -66,17 +65,14 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
* <p>The JobGraph defines the job-wide configuration settings, while each vertex and intermediate
* result define the characteristics of the concrete operation and intermediate data.
*/
-public class JobGraph implements Serializable {
+public class JobGraph implements ExecutionPlan {
private static final long serialVersionUID = 1L;
- private static final ConfigOption<Long> INITIAL_CLIENT_HEARTBEAT_TIMEOUT =
- ConfigOptions.key("initialClientHeartbeatTimeout")
- .longType()
- .defaultValue(Long.MIN_VALUE);
-
// --- job and configuration ---
+ private long initialClientHeartbeatTimeout;
+
/** List of task vertices included in this job graph. */
private final Map<JobVertexID, JobVertex> taskVertices =
new LinkedHashMap<JobVertexID, JobVertex>();
@@ -185,6 +181,7 @@ public class JobGraph implements Serializable {
*
* @return the ID of the job
*/
+ @Override
public JobID getJobID() {
return this.jobID;
}
@@ -199,10 +196,36 @@ public class JobGraph implements Serializable {
*
* @return the name assigned to the job graph
*/
+ @Override
public String getName() {
return this.jobName;
}
+ @Override
+ public boolean isPartialResourceConfigured() {
+ boolean hasVerticesWithUnknownResource = false;
+ boolean hasVerticesWithConfiguredResource = false;
+
+ for (JobVertex jobVertex : getVertices()) {
+ if (jobVertex.getMinResources() == ResourceSpec.UNKNOWN) {
+ hasVerticesWithUnknownResource = true;
+ } else {
+ hasVerticesWithConfiguredResource = true;
+ }
+
+ if (hasVerticesWithUnknownResource && hasVerticesWithConfiguredResource) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return getNumberOfVertices() == 0;
+ }
+
public void setJobConfiguration(Configuration jobConfiguration) {
this.jobConfiguration = jobConfiguration;
}
@@ -213,6 +236,7 @@ public class JobGraph implements Serializable {
*
* @return The configuration object for this job.
*/
+ @Override
public Configuration getJobConfiguration() {
return this.jobConfiguration;
}
@@ -222,6 +246,7 @@ public class JobGraph implements Serializable {
*
* @return ExecutionConfig
*/
+ @Override
public SerializedValue<ExecutionConfig> getSerializedExecutionConfig() {
return serializedExecutionConfig;
}
@@ -230,6 +255,7 @@ public class JobGraph implements Serializable {
this.jobType = type;
}
+ @Override
public JobType getJobType() {
return jobType;
}
@@ -238,6 +264,7 @@ public class JobGraph implements Serializable {
this.dynamic = dynamic;
}
+ @Override
public boolean isDynamic() {
return dynamic;
}
@@ -255,6 +282,7 @@ public class JobGraph implements Serializable {
*
* @param settings The savepoint restore settings.
*/
+ @Override
public void setSavepointRestoreSettings(SavepointRestoreSettings settings) {
this.savepointRestoreSettings = checkNotNull(settings, "Savepoint restore settings");
}
@@ -264,6 +292,7 @@ public class JobGraph implements Serializable {
*
* @return The configured savepoint restore settings.
*/
+ @Override
public SavepointRestoreSettings getSavepointRestoreSettings() {
return savepointRestoreSettings;
}
@@ -371,24 +400,11 @@ public class JobGraph implements Serializable {
*
* @return The snapshot settings
*/
+ @Override
public JobCheckpointingSettings getCheckpointingSettings() {
return snapshotSettings;
}
- /**
- * Checks if the checkpointing was enabled for this job graph.
- *
- * @return true if checkpointing enabled
- */
- public boolean isCheckpointingEnabled() {
-
- if (snapshotSettings == null) {
- return false;
- }
-
- return snapshotSettings.getCheckpointCoordinatorConfiguration().isCheckpointingEnabled();
- }
-
/**
* Searches for a vertex with a matching ID and returns it.
*
@@ -409,6 +425,7 @@ public class JobGraph implements Serializable {
classpaths = paths;
}
+ @Override
public List<URL> getClasspaths() {
return classpaths;
}
@@ -418,6 +435,7 @@ public class JobGraph implements Serializable {
*
* @return The maximum parallelism of this job graph
*/
+ @Override
public int getMaximumParallelism() {
int maxParallelism = -1;
for (JobVertex vertex : taskVertices.values()) {
@@ -551,6 +569,7 @@ public class JobGraph implements Serializable {
*
* @return The list of assigned user jar paths
*/
+ @Override
public List<Path> getUserJars() {
return userJars;
}
@@ -575,6 +594,7 @@ public class JobGraph implements Serializable {
*
* @return The list of assigned user jar paths
*/
+ @Override
public Map<String, DistributedCache.DistributedCacheEntry> getUserArtifacts() {
return userArtifacts;
}
@@ -584,6 +604,7 @@ public class JobGraph implements Serializable {
*
* @param key path of the JAR file required to run the job on a task manager
*/
+ @Override
public void addUserJarBlobKey(PermanentBlobKey key) {
if (key == null) {
throw new IllegalArgumentException();
@@ -608,6 +629,7 @@ public class JobGraph implements Serializable {
*
* @return set of BLOB keys referring to the JAR files required to run this job
*/
+ @Override
public List<PermanentBlobKey> getUserJarBlobKeys() {
return this.userJarBlobKeys;
}
@@ -617,6 +639,7 @@ public class JobGraph implements Serializable {
return "JobGraph(jobId: " + jobID + ")";
}
+ @Override
public void setUserArtifactBlobKey(String entryName, PermanentBlobKey blobKey)
throws IOException {
byte[] serializedBlobKey;
@@ -643,6 +666,7 @@ public class JobGraph implements Serializable {
originalEntry.isZipped));
}
+ @Override
public void writeUserArtifactEntriesToConfiguration() {
for (Map.Entry<String, DistributedCache.DistributedCacheEntry> userArtifact :
userArtifacts.entrySet()) {
@@ -661,10 +685,11 @@ public class JobGraph implements Serializable {
}
public void setInitialClientHeartbeatTimeout(long initialClientHeartbeatTimeout) {
- jobConfiguration.set(INITIAL_CLIENT_HEARTBEAT_TIMEOUT, initialClientHeartbeatTimeout);
+ this.initialClientHeartbeatTimeout = initialClientHeartbeatTimeout;
}
+ @Override
public long getInitialClientHeartbeatTimeout() {
- return jobConfiguration.get(INITIAL_CLIENT_HEARTBEAT_TIMEOUT);
+ return initialClientHeartbeatTimeout;
}
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/streaming/api/graph/ExecutionPlan.java b/flink-runtime/src/main/java/org/apache/flink/streaming/api/graph/ExecutionPlan.java
new file mode 100644
index 00000000000..8b4216fae18
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/streaming/api/graph/ExecutionPlan.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.graph;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.cache.DistributedCache;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.blob.PermanentBlobKey;
+import org.apache.flink.runtime.jobgraph.JobType;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
+import org.apache.flink.util.SerializedValue;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URL;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * An interface representing a general execution plan, which can be implemented by different types
+ * of graphs such as JobGraph and StreamGraph.
+ *
+ * <p>This interface provides methods for accessing general properties of execution plans, such as
+ * the job id and job name.
+ */
+@Internal
+public interface ExecutionPlan extends Serializable {
+
+ /**
+ * Gets the unique identifier of the job.
+ *
+ * @return the job id
+ */
+ JobID getJobID();
+
+ /**
+ * Gets the name of the job.
+ *
+ * @return the job name
+ */
+ String getName();
+
+ /**
+ * Gets the type of the job.
+ *
+ * @return the job type
+ */
+ JobType getJobType();
+
+ /**
+ * Checks if the execution plan is dynamic.
+ *
+ * @return true if the execution plan is dynamic; false otherwise
+ */
+ boolean isDynamic();
+
+ /**
+ * Gets the settings for job checkpointing.
+ *
+ * @return the checkpointing settings
+ */
+ JobCheckpointingSettings getCheckpointingSettings();
+
+ /**
+ * Checks if the execution plan is empty.
+ *
+ * @return true if the plan is empty; false otherwise
+ */
+ boolean isEmpty();
+
+ /**
+ * Gets the initial client heartbeat timeout.
+ *
+ * @return the timeout duration in milliseconds
+ */
+ long getInitialClientHeartbeatTimeout();
+
+ /**
+ * Checks if partial resource configuration is specified.
+ *
+ * @return true if partial resource configuration is set; false otherwise
+ */
+ boolean isPartialResourceConfigured();
+
+ /**
+ * Gets the maximum parallelism level for the job.
+ *
+ * @return the maximum parallelism
+ */
+ int getMaximumParallelism();
+
+ /**
+ * Gets the job configuration.
+ *
+ * @return the job configuration
+ */
+ Configuration getJobConfiguration();
+
+ /**
+ * Gets the user-defined JAR files required for the job.
+ *
+ * @return a list of paths to user JAR files
+ */
+ List<Path> getUserJars();
+
+ /**
+ * Gets the user-defined blob keys corresponding to the JAR files.
+ *
+ * @return a list of permanent blob keys for user JARs
+ */
+ List<PermanentBlobKey> getUserJarBlobKeys();
+
+ /**
+ * Gets the classpath required for the job.
+ *
+ * @return a list of classpath URLs
+ */
+ List<URL> getClasspaths();
+
+ /**
+ * Gets the user artifacts associated with the job.
+ *
+ * @return a map of user artifacts
+ */
+ Map<String, DistributedCache.DistributedCacheEntry> getUserArtifacts();
+
+ /**
+ * Adds a blob key corresponding to a user JAR.
+ *
+ * @param permanentBlobKey the blob key to add
+ */
+ void addUserJarBlobKey(PermanentBlobKey permanentBlobKey);
+
+ /**
+ * Sets a user artifact blob key for a specified user artifact.
+ *
+ * @param artifactName the name of the user artifact
+ * @param blobKey the blob key corresponding to the user artifact
+ * @throws IOException if an error occurs during the operation
+ */
+ void setUserArtifactBlobKey(String artifactName, PermanentBlobKey blobKey) throws IOException;
+
+ /** Writes user artifact entries to the job configuration. */
+ void writeUserArtifactEntriesToConfiguration();
+
+ /**
+ * Gets the settings for restoring from a savepoint.
+ *
+ * @return the savepoint restore settings
+ */
+ SavepointRestoreSettings getSavepointRestoreSettings();
+
+ /**
+ * Sets the settings for restoring from a savepoint.
+ *
+ * @param savepointRestoreSettings the settings for savepoint restoration
+ */
+ void setSavepointRestoreSettings(SavepointRestoreSettings savepointRestoreSettings);
+
+ /**
+ * Checks if the checkpointing was enabled.
+ *
+ * @return true if checkpointing enabled
+ */
+ default boolean isCheckpointingEnabled() {
+ JobCheckpointingSettings checkpointingSettings = getCheckpointingSettings();
+ if (checkpointingSettings == null) {
+ return false;
+ }
+
+ return checkpointingSettings
+ .getCheckpointCoordinatorConfiguration()
+ .isCheckpointingEnabled();
+ }
+
+ /**
+ * Gets the serialized execution configuration.
+ *
+ * @return The serialized execution configuration object
+ */
+ SerializedValue<ExecutionConfig> getSerializedExecutionConfig();
+}