You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ga...@apache.org on 2020/04/24 13:38:05 UTC
[flink] 03/05: [FLINK-17180][runtime] Use SchedulingPipelinedRegion
in RegionPartitionReleaseStrategy
This is an automated email from the ASF dual-hosted git repository.
gary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 95b3c955f115dacb58b9695ae4192f729f5d5662
Author: Gary Yao <ga...@apache.org>
AuthorDate: Tue Apr 21 15:23:40 2020 +0200
[FLINK-17180][runtime] Use SchedulingPipelinedRegion in RegionPartitionReleaseStrategy
Avoid re-computing pipelined regions in the
RegionPartitionReleaseStrategy using the PipelinedRegionComputeUtil.
Instead, rely on the pipelined regions provided by the Topology.
---
.../failover/flip1/PipelinedRegionComputeUtil.java | 20 -----
.../flip1/partitionrelease/PipelinedRegion.java | 69 ---------------
.../PipelinedRegionConsumedBlockingPartitions.java | 51 ------------
.../PipelinedRegionExecutionView.java | 24 +++---
.../RegionPartitionReleaseStrategy.java | 97 ++++------------------
.../RegionPartitionReleaseStrategyTest.java | 38 +--------
.../PipelinedRegionExecutionViewTest.java | 21 +++--
7 files changed, 45 insertions(+), 275 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/PipelinedRegionComputeUtil.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/PipelinedRegionComputeUtil.java
index aa94841..52d96f4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/PipelinedRegionComputeUtil.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/PipelinedRegionComputeUtil.java
@@ -19,9 +19,6 @@
package org.apache.flink.runtime.executiongraph.failover.flip1;
-import org.apache.flink.runtime.executiongraph.failover.flip1.partitionrelease.PipelinedRegion;
-import org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID;
-import org.apache.flink.runtime.scheduler.strategy.SchedulingExecutionVertex;
import org.apache.flink.runtime.topology.BaseTopology;
import org.apache.flink.runtime.topology.Result;
import org.apache.flink.runtime.topology.Vertex;
@@ -34,8 +31,6 @@ import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Map;
import java.util.Set;
-import java.util.function.Function;
-import java.util.stream.Collectors;
/**
* Utility for computing pipelined regions.
@@ -44,21 +39,6 @@ public final class PipelinedRegionComputeUtil {
private static final Logger LOG = LoggerFactory.getLogger(PipelinedRegionComputeUtil.class);
- public static Set<PipelinedRegion> toPipelinedRegionsSet(
- final Set<? extends Set<? extends SchedulingExecutionVertex>> distinctRegions) {
-
- return distinctRegions.stream()
- .map(toExecutionVertexIdSet())
- .map(PipelinedRegion::from)
- .collect(Collectors.toSet());
- }
-
- private static Function<Set<? extends SchedulingExecutionVertex>, Set<ExecutionVertexID>> toExecutionVertexIdSet() {
- return failoverVertices -> failoverVertices.stream()
- .map(SchedulingExecutionVertex::getId)
- .collect(Collectors.toSet());
- }
-
public static <V extends Vertex<?, ?, V, R>, R extends Result<?, ?, V, R>> Set<Set<V>> computePipelinedRegions(
final BaseTopology<?, ?, V, R> topology) {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/PipelinedRegion.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/PipelinedRegion.java
deleted file mode 100644
index 36c042e..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/PipelinedRegion.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.runtime.executiongraph.failover.flip1.partitionrelease;
-
-import org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Set;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/**
- * Set of execution vertices that are connected through pipelined intermediate result partitions.
- */
-public class PipelinedRegion implements Iterable<ExecutionVertexID> {
-
- private final Set<ExecutionVertexID> executionVertexIds;
-
- private PipelinedRegion(final Set<ExecutionVertexID> executionVertexIds) {
- this.executionVertexIds = new HashSet<>(checkNotNull(executionVertexIds));
- }
-
- public static PipelinedRegion from(final Set<ExecutionVertexID> executionVertexIds) {
- return new PipelinedRegion(executionVertexIds);
- }
-
- public static PipelinedRegion from(final ExecutionVertexID... executionVertexIds) {
- return new PipelinedRegion(new HashSet<>(Arrays.asList(executionVertexIds)));
- }
-
- public Set<ExecutionVertexID> getExecutionVertexIds() {
- return executionVertexIds;
- }
-
- public boolean contains(final ExecutionVertexID executionVertexId) {
- return executionVertexIds.contains(executionVertexId);
- }
-
- @Override
- public Iterator<ExecutionVertexID> iterator() {
- return executionVertexIds.iterator();
- }
-
- @Override
- public String toString() {
- return "PipelinedRegion{" +
- "executionVertexIds=" + executionVertexIds +
- '}';
- }
-}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/PipelinedRegionConsumedBlockingPartitions.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/PipelinedRegionConsumedBlockingPartitions.java
deleted file mode 100644
index 5cbce45..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/PipelinedRegionConsumedBlockingPartitions.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.runtime.executiongraph.failover.flip1.partitionrelease;
-
-import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID;
-
-import java.util.Set;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/**
- * A set of intermediate result partitions that are incident to one {@link PipelinedRegion}.
- */
-class PipelinedRegionConsumedBlockingPartitions {
-
- private final PipelinedRegion pipelinedRegion;
-
- private final Set<IntermediateResultPartitionID> consumedBlockingPartitions;
-
- PipelinedRegionConsumedBlockingPartitions(
- final PipelinedRegion pipelinedRegion,
- final Set<IntermediateResultPartitionID> consumedBlockingPartitions) {
- this.pipelinedRegion = checkNotNull(pipelinedRegion);
- this.consumedBlockingPartitions = checkNotNull(consumedBlockingPartitions);
- }
-
- public Set<IntermediateResultPartitionID> getConsumedBlockingPartitions() {
- return consumedBlockingPartitions;
- }
-
- public PipelinedRegion getPipelinedRegion() {
- return pipelinedRegion;
- }
-}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/PipelinedRegionExecutionView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/PipelinedRegionExecutionView.java
index c92fa8b..493d64a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/PipelinedRegionExecutionView.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/PipelinedRegionExecutionView.java
@@ -20,28 +20,32 @@
package org.apache.flink.runtime.executiongraph.failover.flip1.partitionrelease;
import org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID;
+import org.apache.flink.runtime.scheduler.strategy.SchedulingExecutionVertex;
+import org.apache.flink.runtime.scheduler.strategy.SchedulingPipelinedRegion;
+import org.apache.flink.util.IterableUtils;
-import java.util.HashSet;
import java.util.Set;
+import java.util.stream.Collectors;
-import static org.apache.flink.util.Preconditions.checkArgument;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
- * Provides a virtual execution state of a {@link PipelinedRegion}.
+ * Provides a virtual execution state of a {@link SchedulingPipelinedRegion}.
*
* <p>A pipelined region can be either finished or unfinished. It is finished iff. all its
* executions have reached the finished state.
*/
class PipelinedRegionExecutionView {
- private final PipelinedRegion pipelinedRegion;
+ private final SchedulingPipelinedRegion pipelinedRegion;
private final Set<ExecutionVertexID> unfinishedVertices;
- PipelinedRegionExecutionView(final PipelinedRegion pipelinedRegion) {
+ PipelinedRegionExecutionView(final SchedulingPipelinedRegion pipelinedRegion) {
this.pipelinedRegion = checkNotNull(pipelinedRegion);
- this.unfinishedVertices = new HashSet<>(pipelinedRegion.getExecutionVertexIds());
+ this.unfinishedVertices = IterableUtils.toStream(pipelinedRegion.getVertices())
+ .map(SchedulingExecutionVertex::getId)
+ .collect(Collectors.toSet());
}
public boolean isFinished() {
@@ -49,16 +53,16 @@ class PipelinedRegionExecutionView {
}
public void vertexFinished(final ExecutionVertexID executionVertexId) {
- checkArgument(pipelinedRegion.contains(executionVertexId));
+ assertVertexInRegion(executionVertexId);
unfinishedVertices.remove(executionVertexId);
}
public void vertexUnfinished(final ExecutionVertexID executionVertexId) {
- checkArgument(pipelinedRegion.contains(executionVertexId));
+ assertVertexInRegion(executionVertexId);
unfinishedVertices.add(executionVertexId);
}
- public PipelinedRegion getPipelinedRegion() {
- return pipelinedRegion;
+ private void assertVertexInRegion(final ExecutionVertexID executionVertexId) {
+ pipelinedRegion.getVertex(executionVertexId);
}
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/RegionPartitionReleaseStrategy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/RegionPartitionReleaseStrategy.java
index 18d9a4e..f561a6f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/RegionPartitionReleaseStrategy.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/RegionPartitionReleaseStrategy.java
@@ -19,105 +19,56 @@
package org.apache.flink.runtime.executiongraph.failover.flip1.partitionrelease;
-import org.apache.flink.runtime.executiongraph.failover.flip1.PipelinedRegionComputeUtil;
import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID;
import org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID;
import org.apache.flink.runtime.scheduler.strategy.SchedulingExecutionVertex;
+import org.apache.flink.runtime.scheduler.strategy.SchedulingPipelinedRegion;
import org.apache.flink.runtime.scheduler.strategy.SchedulingResultPartition;
import org.apache.flink.runtime.scheduler.strategy.SchedulingTopology;
import org.apache.flink.util.IterableUtils;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
-import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.stream.Collectors;
import static org.apache.flink.util.Preconditions.checkNotNull;
import static org.apache.flink.util.Preconditions.checkState;
/**
- * Releases blocking intermediate result partitions that are incident to a {@link PipelinedRegion},
+ * Releases blocking intermediate result partitions that are incident to a {@link SchedulingPipelinedRegion},
* as soon as the region's execution vertices are finished.
*/
public class RegionPartitionReleaseStrategy implements PartitionReleaseStrategy {
private final SchedulingTopology schedulingTopology;
- private final Map<PipelinedRegion, PipelinedRegionConsumedBlockingPartitions> consumedBlockingPartitionsByRegion = new IdentityHashMap<>();
-
private final Map<ExecutionVertexID, PipelinedRegionExecutionView> regionExecutionViewByVertex = new HashMap<>();
- public RegionPartitionReleaseStrategy(
- final SchedulingTopology schedulingTopology,
- final Set<PipelinedRegion> pipelinedRegions) {
-
+ public RegionPartitionReleaseStrategy(final SchedulingTopology schedulingTopology) {
this.schedulingTopology = checkNotNull(schedulingTopology);
- checkNotNull(pipelinedRegions);
- initConsumedBlockingPartitionsByRegion(pipelinedRegions);
- initRegionExecutionViewByVertex(pipelinedRegions);
- }
-
- private void initConsumedBlockingPartitionsByRegion(final Set<PipelinedRegion> pipelinedRegions) {
- for (PipelinedRegion pipelinedRegion : pipelinedRegions) {
- final PipelinedRegionConsumedBlockingPartitions consumedPartitions = computeConsumedPartitionsOfVertexRegion(pipelinedRegion);
- consumedBlockingPartitionsByRegion.put(pipelinedRegion, consumedPartitions);
- }
+ initRegionExecutionViewByVertex();
}
- private void initRegionExecutionViewByVertex(final Set<PipelinedRegion> pipelinedRegions) {
- for (PipelinedRegion pipelinedRegion : pipelinedRegions) {
+ private void initRegionExecutionViewByVertex() {
+ for (SchedulingPipelinedRegion pipelinedRegion : schedulingTopology.getAllPipelinedRegions()) {
final PipelinedRegionExecutionView regionExecutionView = new PipelinedRegionExecutionView(pipelinedRegion);
- for (ExecutionVertexID executionVertexId : pipelinedRegion) {
- regionExecutionViewByVertex.put(executionVertexId, regionExecutionView);
+ for (SchedulingExecutionVertex executionVertexId : pipelinedRegion.getVertices()) {
+ regionExecutionViewByVertex.put(executionVertexId.getId(), regionExecutionView);
}
}
}
- private PipelinedRegionConsumedBlockingPartitions computeConsumedPartitionsOfVertexRegion(final PipelinedRegion pipelinedRegion) {
- final Set<IntermediateResultPartitionID> resultPartitionsOutsideOfRegion = findResultPartitionsOutsideOfRegion(pipelinedRegion);
- return new PipelinedRegionConsumedBlockingPartitions(pipelinedRegion, resultPartitionsOutsideOfRegion);
- }
-
- private Set<IntermediateResultPartitionID> findResultPartitionsOutsideOfRegion(final PipelinedRegion pipelinedRegion) {
- final Set<SchedulingResultPartition> allConsumedPartitionsInRegion = pipelinedRegion
- .getExecutionVertexIds()
- .stream()
- .map(schedulingTopology::getVertex)
- .flatMap(vertex -> IterableUtils.toStream(vertex.getConsumedResults()))
- .collect(Collectors.toSet());
-
- return filterResultPartitionsOutsideOfRegion(allConsumedPartitionsInRegion, pipelinedRegion);
- }
-
- private static Set<IntermediateResultPartitionID> filterResultPartitionsOutsideOfRegion(
- final Collection<SchedulingResultPartition> resultPartitions,
- final PipelinedRegion pipelinedRegion) {
-
- final Set<IntermediateResultPartitionID> result = new HashSet<>();
- for (final SchedulingResultPartition maybeOutsidePartition : resultPartitions) {
- final SchedulingExecutionVertex producer = maybeOutsidePartition.getProducer();
- if (!pipelinedRegion.contains(producer.getId())) {
- result.add(maybeOutsidePartition.getId());
- }
- }
- return result;
- }
-
@Override
public List<IntermediateResultPartitionID> vertexFinished(final ExecutionVertexID finishedVertex) {
final PipelinedRegionExecutionView regionExecutionView = getPipelinedRegionExecutionViewForVertex(finishedVertex);
regionExecutionView.vertexFinished(finishedVertex);
if (regionExecutionView.isFinished()) {
- final PipelinedRegion pipelinedRegion = getPipelinedRegionForVertex(finishedVertex);
- final PipelinedRegionConsumedBlockingPartitions consumedPartitionsOfVertexRegion = getConsumedBlockingPartitionsForRegion(pipelinedRegion);
- return filterReleasablePartitions(consumedPartitionsOfVertexRegion);
+ final SchedulingPipelinedRegion pipelinedRegion = schedulingTopology.getPipelinedRegionOfVertex(finishedVertex);
+ return filterReleasablePartitions(pipelinedRegion.getConsumedResults());
}
return Collections.emptyList();
}
@@ -135,23 +86,9 @@ public class RegionPartitionReleaseStrategy implements PartitionReleaseStrategy
return pipelinedRegionExecutionView;
}
- private PipelinedRegion getPipelinedRegionForVertex(final ExecutionVertexID executionVertexId) {
- final PipelinedRegionExecutionView pipelinedRegionExecutionView = getPipelinedRegionExecutionViewForVertex(executionVertexId);
- return pipelinedRegionExecutionView.getPipelinedRegion();
- }
-
- private PipelinedRegionConsumedBlockingPartitions getConsumedBlockingPartitionsForRegion(final PipelinedRegion pipelinedRegion) {
- final PipelinedRegionConsumedBlockingPartitions pipelinedRegionConsumedBlockingPartitions = consumedBlockingPartitionsByRegion.get(pipelinedRegion);
- checkState(pipelinedRegionConsumedBlockingPartitions != null,
- "Consumed partitions not found for pipelined region %s", pipelinedRegion);
- checkState(pipelinedRegionConsumedBlockingPartitions.getPipelinedRegion() == pipelinedRegion);
- return pipelinedRegionConsumedBlockingPartitions;
- }
-
- private List<IntermediateResultPartitionID> filterReleasablePartitions(final PipelinedRegionConsumedBlockingPartitions consumedPartitionsOfVertexRegion) {
- return consumedPartitionsOfVertexRegion
- .getConsumedBlockingPartitions()
- .stream()
+ private List<IntermediateResultPartitionID> filterReleasablePartitions(final Iterable<? extends SchedulingResultPartition> schedulingResultPartitions) {
+ return IterableUtils.toStream(schedulingResultPartitions)
+ .map(SchedulingResultPartition::getId)
.filter(this::areConsumerRegionsFinished)
.collect(Collectors.toList());
}
@@ -175,13 +112,7 @@ public class RegionPartitionReleaseStrategy implements PartitionReleaseStrategy
@Override
public PartitionReleaseStrategy createInstance(final SchedulingTopology schedulingStrategy) {
-
- final Set<? extends Set<SchedulingExecutionVertex>> distinctRegions =
- PipelinedRegionComputeUtil.computePipelinedRegions(schedulingStrategy);
-
- return new RegionPartitionReleaseStrategy(
- schedulingStrategy,
- PipelinedRegionComputeUtil.toPipelinedRegionsSet(distinctRegions));
+ return new RegionPartitionReleaseStrategy(schedulingStrategy);
}
}
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/RegionPartitionReleaseStrategyTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/RegionPartitionReleaseStrategyTest.java
index 1b9ca7a..4aa8782 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/RegionPartitionReleaseStrategyTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/RegionPartitionReleaseStrategyTest.java
@@ -19,7 +19,6 @@
package org.apache.flink.runtime.executiongraph;
-import org.apache.flink.runtime.executiongraph.failover.flip1.partitionrelease.PipelinedRegion;
import org.apache.flink.runtime.executiongraph.failover.flip1.partitionrelease.RegionPartitionReleaseStrategy;
import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID;
@@ -32,10 +31,7 @@ import org.apache.flink.util.TestLogger;
import org.junit.Before;
import org.junit.Test;
-import java.util.Arrays;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.empty;
@@ -60,15 +56,10 @@ public class RegionPartitionReleaseStrategyTest extends TestLogger {
final List<TestingSchedulingExecutionVertex> consumers = testingSchedulingTopology.addExecutionVertices().finish();
final List<TestingSchedulingResultPartition> resultPartitions = testingSchedulingTopology.connectPointwise(producers, consumers).finish();
- final ExecutionVertexID onlyProducerVertexId = producers.get(0).getId();
final ExecutionVertexID onlyConsumerVertexId = consumers.get(0).getId();
final IntermediateResultPartitionID onlyResultPartitionId = resultPartitions.get(0).getId();
- final Set<PipelinedRegion> pipelinedRegions = pipelinedRegionsSet(
- PipelinedRegion.from(onlyProducerVertexId),
- PipelinedRegion.from(onlyConsumerVertexId));
-
- final RegionPartitionReleaseStrategy regionPartitionReleaseStrategy = new RegionPartitionReleaseStrategy(testingSchedulingTopology, pipelinedRegions);
+ final RegionPartitionReleaseStrategy regionPartitionReleaseStrategy = new RegionPartitionReleaseStrategy(testingSchedulingTopology);
final List<IntermediateResultPartitionID> partitionsToRelease = regionPartitionReleaseStrategy.vertexFinished(onlyConsumerVertexId);
assertThat(partitionsToRelease, contains(onlyResultPartitionId));
@@ -82,16 +73,11 @@ public class RegionPartitionReleaseStrategyTest extends TestLogger {
final List<TestingSchedulingResultPartition> sourceResultPartitions = testingSchedulingTopology.connectAllToAll(sourceVertices, intermediateVertices).finish();
testingSchedulingTopology.connectAllToAll(intermediateVertices, sinkVertices).withResultPartitionType(ResultPartitionType.PIPELINED).finish();
- final ExecutionVertexID onlySourceVertexId = sourceVertices.get(0).getId();
final ExecutionVertexID onlyIntermediateVertexId = intermediateVertices.get(0).getId();
final ExecutionVertexID onlySinkVertexId = sinkVertices.get(0).getId();
final IntermediateResultPartitionID onlySourceResultPartitionId = sourceResultPartitions.get(0).getId();
- final Set<PipelinedRegion> pipelinedRegions = pipelinedRegionsSet(
- PipelinedRegion.from(onlySourceVertexId),
- PipelinedRegion.from(onlyIntermediateVertexId, onlySinkVertexId));
-
- final RegionPartitionReleaseStrategy regionPartitionReleaseStrategy = new RegionPartitionReleaseStrategy(testingSchedulingTopology, pipelinedRegions);
+ final RegionPartitionReleaseStrategy regionPartitionReleaseStrategy = new RegionPartitionReleaseStrategy(testingSchedulingTopology);
regionPartitionReleaseStrategy.vertexFinished(onlyIntermediateVertexId);
final List<IntermediateResultPartitionID> partitionsToRelease = regionPartitionReleaseStrategy.vertexFinished(onlySinkVertexId);
@@ -104,15 +90,9 @@ public class RegionPartitionReleaseStrategyTest extends TestLogger {
final List<TestingSchedulingExecutionVertex> consumers = testingSchedulingTopology.addExecutionVertices().withParallelism(2).finish();
testingSchedulingTopology.connectAllToAll(producers, consumers).finish();
- final ExecutionVertexID onlyProducerVertexId = producers.get(0).getId();
final ExecutionVertexID consumerVertex1 = consumers.get(0).getId();
- final ExecutionVertexID consumerVertex2 = consumers.get(1).getId();
-
- final Set<PipelinedRegion> pipelinedRegions = pipelinedRegionsSet(
- PipelinedRegion.from(onlyProducerVertexId),
- PipelinedRegion.from(consumerVertex1, consumerVertex2));
- final RegionPartitionReleaseStrategy regionPartitionReleaseStrategy = new RegionPartitionReleaseStrategy(testingSchedulingTopology, pipelinedRegions);
+ final RegionPartitionReleaseStrategy regionPartitionReleaseStrategy = new RegionPartitionReleaseStrategy(testingSchedulingTopology);
final List<IntermediateResultPartitionID> partitionsToRelease = regionPartitionReleaseStrategy.vertexFinished(consumerVertex1);
assertThat(partitionsToRelease, is(empty()));
@@ -124,15 +104,10 @@ public class RegionPartitionReleaseStrategyTest extends TestLogger {
final List<TestingSchedulingExecutionVertex> consumers = testingSchedulingTopology.addExecutionVertices().withParallelism(2).finish();
testingSchedulingTopology.connectAllToAll(producers, consumers).finish();
- final ExecutionVertexID onlyProducerVertexId = producers.get(0).getId();
final ExecutionVertexID consumerVertex1 = consumers.get(0).getId();
final ExecutionVertexID consumerVertex2 = consumers.get(1).getId();
- final Set<PipelinedRegion> pipelinedRegions = pipelinedRegionsSet(
- PipelinedRegion.from(onlyProducerVertexId),
- PipelinedRegion.from(consumerVertex1, consumerVertex2));
-
- final RegionPartitionReleaseStrategy regionPartitionReleaseStrategy = new RegionPartitionReleaseStrategy(testingSchedulingTopology, pipelinedRegions);
+ final RegionPartitionReleaseStrategy regionPartitionReleaseStrategy = new RegionPartitionReleaseStrategy(testingSchedulingTopology);
regionPartitionReleaseStrategy.vertexFinished(consumerVertex1);
regionPartitionReleaseStrategy.vertexFinished(consumerVertex2);
@@ -141,9 +116,4 @@ public class RegionPartitionReleaseStrategyTest extends TestLogger {
final List<IntermediateResultPartitionID> partitionsToRelease = regionPartitionReleaseStrategy.vertexFinished(consumerVertex1);
assertThat(partitionsToRelease, is(empty()));
}
-
- private static Set<PipelinedRegion> pipelinedRegionsSet(final PipelinedRegion... pipelinedRegions) {
- return new HashSet<>(Arrays.asList(pipelinedRegions));
- }
-
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/PipelinedRegionExecutionViewTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/PipelinedRegionExecutionViewTest.java
index a6960ef..33820b5 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/PipelinedRegionExecutionViewTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/flip1/partitionrelease/PipelinedRegionExecutionViewTest.java
@@ -21,10 +21,14 @@ package org.apache.flink.runtime.executiongraph.failover.flip1.partitionrelease;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID;
+import org.apache.flink.runtime.scheduler.strategy.TestingSchedulingExecutionVertex;
+import org.apache.flink.runtime.scheduler.strategy.TestingSchedulingPipelinedRegion;
import org.apache.flink.util.TestLogger;
import org.junit.Test;
+import java.util.Collections;
+
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -35,18 +39,21 @@ public class PipelinedRegionExecutionViewTest extends TestLogger {
private static final ExecutionVertexID TEST_EXECUTION_VERTEX_ID = new ExecutionVertexID(new JobVertexID(), 0);
+ private static final TestingSchedulingPipelinedRegion TEST_PIPELINED_REGION = new TestingSchedulingPipelinedRegion(Collections.singleton(
+ new TestingSchedulingExecutionVertex(
+ TEST_EXECUTION_VERTEX_ID.getJobVertexId(),
+ TEST_EXECUTION_VERTEX_ID.getSubtaskIndex())));
+
@Test
public void regionIsUnfinishedIfNotAllVerticesAreFinished() {
- final PipelinedRegion pipelinedRegion = PipelinedRegion.from(TEST_EXECUTION_VERTEX_ID);
- final PipelinedRegionExecutionView pipelinedRegionExecutionView = new PipelinedRegionExecutionView(pipelinedRegion);
+ final PipelinedRegionExecutionView pipelinedRegionExecutionView = new PipelinedRegionExecutionView(TEST_PIPELINED_REGION);
assertFalse(pipelinedRegionExecutionView.isFinished());
}
@Test
public void regionIsFinishedIfAllVerticesAreFinished() {
- final PipelinedRegion pipelinedRegion = PipelinedRegion.from(TEST_EXECUTION_VERTEX_ID);
- final PipelinedRegionExecutionView pipelinedRegionExecutionView = new PipelinedRegionExecutionView(pipelinedRegion);
+ final PipelinedRegionExecutionView pipelinedRegionExecutionView = new PipelinedRegionExecutionView(TEST_PIPELINED_REGION);
pipelinedRegionExecutionView.vertexFinished(TEST_EXECUTION_VERTEX_ID);
@@ -55,8 +62,7 @@ public class PipelinedRegionExecutionViewTest extends TestLogger {
@Test
public void vertexCanBeUnfinished() {
- final PipelinedRegion pipelinedRegion = PipelinedRegion.from(TEST_EXECUTION_VERTEX_ID);
- final PipelinedRegionExecutionView pipelinedRegionExecutionView = new PipelinedRegionExecutionView(pipelinedRegion);
+ final PipelinedRegionExecutionView pipelinedRegionExecutionView = new PipelinedRegionExecutionView(TEST_PIPELINED_REGION);
pipelinedRegionExecutionView.vertexFinished(TEST_EXECUTION_VERTEX_ID);
pipelinedRegionExecutionView.vertexUnfinished(TEST_EXECUTION_VERTEX_ID);
@@ -66,8 +72,7 @@ public class PipelinedRegionExecutionViewTest extends TestLogger {
@Test(expected = IllegalArgumentException.class)
public void finishingUnknownVertexThrowsException() {
- final PipelinedRegion from = PipelinedRegion.from(TEST_EXECUTION_VERTEX_ID);
- final PipelinedRegionExecutionView pipelinedRegionExecutionView = new PipelinedRegionExecutionView(from);
+ final PipelinedRegionExecutionView pipelinedRegionExecutionView = new PipelinedRegionExecutionView(TEST_PIPELINED_REGION);
final ExecutionVertexID unknownVertexId = new ExecutionVertexID(new JobVertexID(), 0);
pipelinedRegionExecutionView.vertexFinished(unknownVertexId);