You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2019/07/10 09:36:07 UTC
[flink] 06/16: [FLINK-12763][runtime] Requests slots with
ResourceProfiles that are converted from ResourceSpecs.
This is an automated email from the ASF dual-hosted git repository.
sewen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit d5520a3207fb6ea9a9505a903cf24d0014d75545
Author: Xintong Song <to...@gmail.com>
AuthorDate: Thu Jul 4 23:24:44 2019 +0800
[FLINK-12763][runtime] Requests slots with ResourceProfiles that are converted from ResourceSpecs.
---
.../flink/runtime/clusterframework/types/ResourceProfile.java | 2 +-
.../runtime/executiongraph/AccessExecutionJobVertex.java | 8 ++++++++
.../runtime/executiongraph/ArchivedExecutionJobVertex.java | 11 +++++++++++
.../org/apache/flink/runtime/executiongraph/Execution.java | 3 +--
.../flink/runtime/executiongraph/ExecutionJobVertex.java | 9 +++++++++
.../apache/flink/runtime/executiongraph/ExecutionVertex.java | 5 +++++
.../job/SubtaskExecutionAttemptDetailsHandlerTest.java | 2 ++
.../legacy/utils/ArchivedExecutionJobVertexBuilder.java | 2 ++
8 files changed, 39 insertions(+), 3 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceProfile.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceProfile.java
index 21eb9b9..0d9eca3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceProfile.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceProfile.java
@@ -325,7 +325,7 @@ public class ResourceProfile implements Serializable, Comparable<ResourceProfile
'}';
}
- static ResourceProfile fromResourceSpec(ResourceSpec resourceSpec, int networkMemory) {
+ public static ResourceProfile fromResourceSpec(ResourceSpec resourceSpec, int networkMemory) {
Map<String, Resource> copiedExtendedResources = new HashMap<>(resourceSpec.getExtendedResources());
return new ResourceProfile(
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionJobVertex.java
index 43b5889..523eac8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionJobVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionJobVertex.java
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.executiongraph;
import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.jobgraph.JobVertexID;
@@ -47,6 +48,13 @@ public interface AccessExecutionJobVertex {
int getMaxParallelism();
/**
+ * Returns the resource profile for this job vertex.
+ *
+ * @return resource profile for this job vertex.
+ */
+ ResourceProfile getResourceProfile();
+
+ /**
* Returns the {@link JobVertexID} for this job vertex.
*
* @return JobVertexID for this job vertex.
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionJobVertex.java
index 6b54760..cc2c7fe 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionJobVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionJobVertex.java
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.executiongraph;
import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.jobgraph.JobVertexID;
@@ -38,6 +39,8 @@ public class ArchivedExecutionJobVertex implements AccessExecutionJobVertex, Ser
private final int maxParallelism;
+ private final ResourceProfile resourceProfile;
+
private final StringifiedAccumulatorResult[] archivedUserAccumulators;
public ArchivedExecutionJobVertex(ExecutionJobVertex jobVertex) {
@@ -52,6 +55,7 @@ public class ArchivedExecutionJobVertex implements AccessExecutionJobVertex, Ser
this.name = jobVertex.getJobVertex().getName();
this.parallelism = jobVertex.getParallelism();
this.maxParallelism = jobVertex.getMaxParallelism();
+ this.resourceProfile = jobVertex.getResourceProfile();
}
public ArchivedExecutionJobVertex(
@@ -60,12 +64,14 @@ public class ArchivedExecutionJobVertex implements AccessExecutionJobVertex, Ser
String name,
int parallelism,
int maxParallelism,
+ ResourceProfile resourceProfile,
StringifiedAccumulatorResult[] archivedUserAccumulators) {
this.taskVertices = taskVertices;
this.id = id;
this.name = name;
this.parallelism = parallelism;
this.maxParallelism = maxParallelism;
+ this.resourceProfile = resourceProfile;
this.archivedUserAccumulators = archivedUserAccumulators;
}
@@ -89,6 +95,11 @@ public class ArchivedExecutionJobVertex implements AccessExecutionJobVertex, Ser
}
@Override
+ public ResourceProfile getResourceProfile() {
+ return resourceProfile;
+ }
+
+ @Override
public JobVertexID getJobVertexId() {
return id;
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index 4939e61..46e1e7f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -31,7 +31,6 @@ import org.apache.flink.runtime.checkpoint.CheckpointType;
import org.apache.flink.runtime.checkpoint.JobManagerTaskRestore;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.clusterframework.types.SlotProfile;
import org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor;
import org.apache.flink.runtime.concurrent.FutureUtils;
@@ -577,7 +576,7 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
slotRequestId,
toSchedule,
new SlotProfile(
- ResourceProfile.UNKNOWN,
+ vertex.getResourceProfile(),
preferredLocations,
previousAllocationIDs,
allPreviousExecutionGraphAllocationIds),
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
index 06d597b..ba9b2e1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
@@ -35,6 +35,7 @@ import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
import org.apache.flink.runtime.blob.BlobWriter;
import org.apache.flink.runtime.blob.PermanentBlobKey;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.concurrent.FutureUtils;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.jobgraph.IntermediateDataSet;
@@ -127,6 +128,8 @@ public class ExecutionJobVertex implements AccessExecutionJobVertex, Archiveable
private int maxParallelism;
+ private final ResourceProfile resourceProfile;
+
/**
* Either store a serialized task information, which is for all sub tasks the same,
* or the permanent blob key of the offloaded task information BLOB containing
@@ -193,6 +196,7 @@ public class ExecutionJobVertex implements AccessExecutionJobVertex, Archiveable
}
this.parallelism = numTaskVertices;
+ this.resourceProfile = ResourceProfile.fromResourceSpec(jobVertex.getMinResources(), 0);
this.taskVertices = new ExecutionVertex[numTaskVertices];
this.operatorIDs = Collections.unmodifiableList(jobVertex.getOperatorIDs());
@@ -334,6 +338,11 @@ public class ExecutionJobVertex implements AccessExecutionJobVertex, Archiveable
return maxParallelism;
}
+ @Override
+ public ResourceProfile getResourceProfile() {
+ return resourceProfile;
+ }
+
public boolean isMaxParallelismConfigured() {
return maxParallelismConfigured;
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
index 349aaff..9848c8e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
@@ -28,6 +28,7 @@ import org.apache.flink.core.io.InputSplit;
import org.apache.flink.core.io.InputSplitAssigner;
import org.apache.flink.runtime.JobException;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
import org.apache.flink.runtime.jobgraph.DistributionPattern;
@@ -227,6 +228,10 @@ public class ExecutionVertex implements AccessExecutionVertex, Archiveable<Archi
return this.jobVertex.getMaxParallelism();
}
+ public ResourceProfile getResourceProfile() {
+ return this.jobVertex.getResourceProfile();
+ }
+
@Override
public int getParallelSubtaskIndex() {
return this.subTaskIndex;
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptDetailsHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptDetailsHandlerTest.java
index ff0ba4a..e452cc7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptDetailsHandlerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptDetailsHandlerTest.java
@@ -23,6 +23,7 @@ import org.apache.flink.api.common.time.Time;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.MetricOptions;
import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.executiongraph.ArchivedExecution;
import org.apache.flink.runtime.executiongraph.ArchivedExecutionJobVertex;
@@ -108,6 +109,7 @@ public class SubtaskExecutionAttemptDetailsHandlerTest extends TestLogger {
"test",
1,
1,
+ ResourceProfile.UNKNOWN,
emptyAccumulators);
// Change some fields so we can make it different from other sub tasks.
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/utils/ArchivedExecutionJobVertexBuilder.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/utils/ArchivedExecutionJobVertexBuilder.java
index 814c4db..d8493ad 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/utils/ArchivedExecutionJobVertexBuilder.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/utils/ArchivedExecutionJobVertexBuilder.java
@@ -19,6 +19,7 @@
package org.apache.flink.runtime.rest.handler.legacy.utils;
import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.executiongraph.ArchivedExecutionJobVertex;
import org.apache.flink.runtime.executiongraph.ArchivedExecutionVertex;
import org.apache.flink.runtime.jobgraph.JobVertexID;
@@ -78,6 +79,7 @@ public class ArchivedExecutionJobVertexBuilder {
name != null ? name : "task_" + RANDOM.nextInt(),
parallelism,
maxParallelism,
+ ResourceProfile.UNKNOWN,
archivedUserAccumulators != null ? archivedUserAccumulators : new StringifiedAccumulatorResult[0]
);
}