You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by xt...@apache.org on 2021/07/05 03:04:10 UTC

[flink] branch master updated (69e1031 -> 25fdf54)

This is an automated email from the ASF dual-hosted git repository.

xtsong pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.


    from 69e1031  [hotfix]: Fix broken javaDoc inline link.
     new d0846bb  [hotfix][core] Fix typo in the Builder of ResourceSpec
     new bc5f36d  [hotfix][core] Check the cpu and heap memory to be positive when building ResourceSpec
     new 0544ddd  [FLINK-21925][core] Introduce public SlotSharingGroup
     new 7871a58  [FLINK-21925][core] Introduce SlotSharingGroupUtils which contains utility for SlotSharingGroup
     new 2e86133  [FLINK-21925][core] Introduce #slotSharingGroup(SlotSharingGroup) for configuring slot sharing group with its resource
     new 3f36bbf  [FLINK-21925][core] Introduce StreamExecutionEnvironment#registerSlotSharingGroup
     new 25fdf54  [FLINK-21925][core] Introduce fine-grained.shuffle-mode.all-blocking to avoid resource deadlock in batch jobs that apply fine-grained resource management

The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../flink/api/common/operators/ResourceSpec.java   |  17 +-
 .../api/common/operators/SlotSharingGroup.java     | 235 +++++++++++++++++++++
 .../operators/util/SlotSharingGroupUtils.java      |  54 +++++
 .../org/apache/flink/api/dag/Transformation.java   |  31 ++-
 .../apache/flink/configuration/ClusterOptions.java |   8 +
 .../api/common/operators/SlotSharingGroupTest.java |  84 ++++++++
 .../operators/util/SlotSharingGroupUtilsTest.java  |  56 +++++
 ...st_stream_execution_environment_completeness.py |   3 +-
 .../apache/flink/python/util/PythonConfigUtil.java |   4 +-
 .../flink/runtime/dispatcher/DispatcherTest.java   |   2 +-
 .../streaming/api/datastream/DataStreamSink.java   |  19 ++
 .../api/datastream/SingleOutputStreamOperator.java |  19 ++
 .../environment/StreamExecutionEnvironment.java    |  58 ++++-
 .../flink/streaming/api/graph/StreamGraph.java     |   5 +
 .../streaming/api/graph/StreamGraphGenerator.java  |  46 +++-
 .../api/StreamExecutionEnvironmentTest.java        |  44 ++++
 .../api/graph/StreamGraphGeneratorTest.java        |  57 +++++
 .../api/scala/StreamExecutionEnvironment.scala     |  17 ++
 18 files changed, 741 insertions(+), 18 deletions(-)
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/operators/util/SlotSharingGroupUtils.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/common/operators/SlotSharingGroupTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/common/operators/util/SlotSharingGroupUtilsTest.java

[flink] 04/07: [FLINK-21925][core] Introduce SlotSharingGroupUtils which contains utility for SlotSharingGroup

Posted by xt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

xtsong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 7871a580f4972762cbd85c8588ae9fa46a816574
Author: Yangze Guo <ka...@gmail.com>
AuthorDate: Wed Jun 30 18:37:29 2021 +0800

    [FLINK-21925][core] Introduce SlotSharingGroupUtils which contains utility for SlotSharingGroup
---
 .../flink/api/common/operators/ResourceSpec.java   |  4 ++
 .../operators/util/SlotSharingGroupUtils.java      | 54 +++++++++++++++++++++
 .../operators/util/SlotSharingGroupUtilsTest.java  | 56 ++++++++++++++++++++++
 3 files changed, 114 insertions(+)

diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/ResourceSpec.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/ResourceSpec.java
index 6b24b66..9990edd 100755
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/ResourceSpec.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/ResourceSpec.java
@@ -332,6 +332,10 @@ public final class ResourceSpec implements Serializable {
         return new Builder(new CPUResource(cpuCores), MemorySize.ofMebiBytes(taskHeapMemoryMB));
     }
 
+    public static Builder newBuilder(double cpuCores, MemorySize taskHeapMemory) {
+        return new Builder(new CPUResource(cpuCores), taskHeapMemory);
+    }
+
     /** Builder for the {@link ResourceSpec}. */
     public static class Builder {
 
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/util/SlotSharingGroupUtils.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/util/SlotSharingGroupUtils.java
new file mode 100644
index 0000000..83c9f96
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/util/SlotSharingGroupUtils.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.operators.util;
+
+import org.apache.flink.api.common.operators.ResourceSpec;
+import org.apache.flink.api.common.operators.SlotSharingGroup;
+import org.apache.flink.api.common.resources.ExternalResource;
+import org.apache.flink.util.Preconditions;
+
+import java.util.stream.Collectors;
+
+/** Utils for {@link SlotSharingGroup}. */
+public class SlotSharingGroupUtils {
+    public static ResourceSpec extractResourceSpec(SlotSharingGroup slotSharingGroup) {
+        if (!slotSharingGroup.getCpuCores().isPresent()) {
+            return ResourceSpec.UNKNOWN;
+        }
+
+        Preconditions.checkState(slotSharingGroup.getCpuCores().isPresent());
+        Preconditions.checkState(slotSharingGroup.getTaskHeapMemory().isPresent());
+        Preconditions.checkState(slotSharingGroup.getTaskOffHeapMemory().isPresent());
+        Preconditions.checkState(slotSharingGroup.getManagedMemory().isPresent());
+
+        return ResourceSpec.newBuilder(
+                        slotSharingGroup.getCpuCores().get(),
+                        slotSharingGroup.getTaskHeapMemory().get())
+                .setTaskOffHeapMemory(slotSharingGroup.getTaskOffHeapMemory().get())
+                .setManagedMemory(slotSharingGroup.getManagedMemory().get())
+                .setExtendedResources(
+                        slotSharingGroup.getExternalResources().entrySet().stream()
+                                .map(
+                                        entry ->
+                                                new ExternalResource(
+                                                        entry.getKey(), entry.getValue()))
+                                .collect(Collectors.toList()))
+                .build();
+    }
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/operators/util/SlotSharingGroupUtilsTest.java b/flink-core/src/test/java/org/apache/flink/api/common/operators/util/SlotSharingGroupUtilsTest.java
new file mode 100644
index 0000000..de72037
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/common/operators/util/SlotSharingGroupUtilsTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.operators.util;
+
+import org.apache.flink.api.common.operators.ResourceSpec;
+import org.apache.flink.api.common.operators.SlotSharingGroup;
+import org.apache.flink.api.common.resources.ExternalResource;
+
+import org.junit.Test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+/** Tests for {@link SlotSharingGroupUtils}. */
+public class SlotSharingGroupUtilsTest {
+    @Test
+    public void testCovertToResourceSpec() {
+        final ExternalResource gpu = new ExternalResource("gpu", 1);
+        final ResourceSpec resourceSpec =
+                ResourceSpec.newBuilder(1.0, 100)
+                        .setManagedMemoryMB(200)
+                        .setTaskOffHeapMemoryMB(300)
+                        .setExtendedResource(gpu)
+                        .build();
+        final SlotSharingGroup slotSharingGroup1 =
+                SlotSharingGroup.newBuilder("ssg")
+                        .setCpuCores(resourceSpec.getCpuCores().getValue().doubleValue())
+                        .setTaskHeapMemory(resourceSpec.getTaskHeapMemory())
+                        .setTaskOffHeapMemory(resourceSpec.getTaskOffHeapMemory())
+                        .setManagedMemory(resourceSpec.getManagedMemory())
+                        .setExternalResource(gpu.getName(), gpu.getValue().doubleValue())
+                        .build();
+        final SlotSharingGroup slotSharingGroup2 = SlotSharingGroup.newBuilder("ssg").build();
+
+        assertThat(SlotSharingGroupUtils.extractResourceSpec(slotSharingGroup1), is(resourceSpec));
+        assertThat(
+                SlotSharingGroupUtils.extractResourceSpec(slotSharingGroup2),
+                is(ResourceSpec.UNKNOWN));
+    }
+}

[flink] 05/07: [FLINK-21925][core] Introduce #slotSharingGroup(SlotSharingGroup) for configuring slot sharing group with its resource

Posted by xt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

xtsong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 2e86133703bbc9f7876a0534c584be11ef120fd8
Author: Yangze Guo <ka...@gmail.com>
AuthorDate: Fri Jun 25 14:53:24 2021 +0800

    [FLINK-21925][core] Introduce #slotSharingGroup(SlotSharingGroup) for configuring slot sharing group with its resource
---
 .../org/apache/flink/api/dag/Transformation.java   | 31 +++++++++---
 .../apache/flink/python/util/PythonConfigUtil.java |  4 +-
 .../streaming/api/datastream/DataStreamSink.java   | 19 ++++++++
 .../api/datastream/SingleOutputStreamOperator.java | 19 ++++++++
 .../streaming/api/graph/StreamGraphGenerator.java  | 46 +++++++++++++++--
 .../api/graph/StreamGraphGeneratorTest.java        | 57 ++++++++++++++++++++++
 6 files changed, 163 insertions(+), 13 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/dag/Transformation.java b/flink-core/src/main/java/org/apache/flink/api/dag/Transformation.java
index 0334b07..915bafb 100644
--- a/flink-core/src/main/java/org/apache/flink/api/dag/Transformation.java
+++ b/flink-core/src/main/java/org/apache/flink/api/dag/Transformation.java
@@ -21,6 +21,7 @@ package org.apache.flink.api.dag;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.functions.InvalidTypesException;
 import org.apache.flink.api.common.operators.ResourceSpec;
+import org.apache.flink.api.common.operators.SlotSharingGroup;
 import org.apache.flink.api.common.operators.util.OperatorValidationUtils;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.typeutils.MissingTypeInfo;
@@ -167,7 +168,7 @@ public abstract class Transformation<T> {
 
     protected long bufferTimeout = -1;
 
-    private String slotSharingGroup;
+    private Optional<SlotSharingGroup> slotSharingGroup;
 
     @Nullable private String coLocationGroupKey;
 
@@ -184,7 +185,7 @@ public abstract class Transformation<T> {
         this.name = Preconditions.checkNotNull(name);
         this.outputType = outputType;
         this.parallelism = parallelism;
-        this.slotSharingGroup = null;
+        this.slotSharingGroup = Optional.empty();
     }
 
     /** Returns the unique ID of this {@code Transformation}. */
@@ -390,11 +391,11 @@ public abstract class Transformation<T> {
     }
 
     /**
-     * Returns the slot sharing group of this transformation.
+     * Returns the slot sharing group of this transformation if present.
      *
-     * @see #setSlotSharingGroup(String)
+     * @see #setSlotSharingGroup(SlotSharingGroup)
      */
-    public String getSlotSharingGroup() {
+    public Optional<SlotSharingGroup> getSlotSharingGroup() {
         return slotSharingGroup;
     }
 
@@ -405,10 +406,24 @@ public abstract class Transformation<T> {
      * <p>Initially, an operation is in the default slot sharing group. This can be explicitly set
      * using {@code setSlotSharingGroup("default")}.
      *
-     * @param slotSharingGroup The slot sharing group name.
+     * @param slotSharingGroupName The slot sharing group's name.
      */
-    public void setSlotSharingGroup(String slotSharingGroup) {
-        this.slotSharingGroup = slotSharingGroup;
+    public void setSlotSharingGroup(String slotSharingGroupName) {
+        this.slotSharingGroup =
+                Optional.of(SlotSharingGroup.newBuilder(slotSharingGroupName).build());
+    }
+
+    /**
+     * Sets the slot sharing group of this transformation. Parallel instances of operations that are
+     * in the same slot sharing group will be co-located in the same TaskManager slot, if possible.
+     *
+     * <p>Initially, an operation is in the default slot sharing group. This can be explicitly set
+     * with constructing a {@link SlotSharingGroup} with name {@code "default"}.
+     *
+     * @param slotSharingGroup which contains name and its resource spec.
+     */
+    public void setSlotSharingGroup(SlotSharingGroup slotSharingGroup) {
+        this.slotSharingGroup = Optional.of(slotSharingGroup);
     }
 
     /**
diff --git a/flink-python/src/main/java/org/apache/flink/python/util/PythonConfigUtil.java b/flink-python/src/main/java/org/apache/flink/python/util/PythonConfigUtil.java
index 8181266..cdba06e 100644
--- a/flink-python/src/main/java/org/apache/flink/python/util/PythonConfigUtil.java
+++ b/flink-python/src/main/java/org/apache/flink/python/util/PythonConfigUtil.java
@@ -215,7 +215,9 @@ public class PythonConfigUtil {
 
     private static void chainTransformation(
             Transformation<?> firstTransformation, Transformation<?> secondTransformation) {
-        firstTransformation.setSlotSharingGroup(secondTransformation.getSlotSharingGroup());
+        secondTransformation
+                .getSlotSharingGroup()
+                .ifPresent(firstTransformation::setSlotSharingGroup);
         firstTransformation.setCoLocationGroupKey(secondTransformation.getCoLocationGroupKey());
         firstTransformation.setParallelism(secondTransformation.getParallelism());
     }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
index a3da7c6..eea988e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
@@ -21,6 +21,7 @@ import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.Public;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.operators.ResourceSpec;
+import org.apache.flink.api.common.operators.SlotSharingGroup;
 import org.apache.flink.api.connector.sink.Sink;
 import org.apache.flink.streaming.api.operators.ChainingStrategy;
 import org.apache.flink.streaming.api.operators.StreamSink;
@@ -205,4 +206,22 @@ public class DataStreamSink<T> {
         transformation.setSlotSharingGroup(slotSharingGroup);
         return this;
     }
+
+    /**
+     * Sets the slot sharing group of this operation. Parallel instances of operations that are in
+     * the same slot sharing group will be co-located in the same TaskManager slot, if possible.
+     *
+     * <p>Operations inherit the slot sharing group of input operations if all input operations are
+     * in the same slot sharing group and no slot sharing group was explicitly specified.
+     *
+     * <p>Initially an operation is in the default slot sharing group. An operation can be put into
+     * the default group explicitly by setting the slot sharing group with name {@code "default"}.
+     *
+     * @param slotSharingGroup which contains name and its resource spec.
+     */
+    @PublicEvolving
+    public DataStreamSink<T> slotSharingGroup(SlotSharingGroup slotSharingGroup) {
+        transformation.setSlotSharingGroup(slotSharingGroup);
+        return this;
+    }
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
index 6b77c24..7b64d21 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -21,6 +21,7 @@ import org.apache.flink.annotation.Public;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.functions.InvalidTypesException;
 import org.apache.flink.api.common.operators.ResourceSpec;
+import org.apache.flink.api.common.operators.SlotSharingGroup;
 import org.apache.flink.api.common.operators.util.OperatorValidationUtils;
 import org.apache.flink.api.common.typeinfo.TypeHint;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
@@ -374,6 +375,24 @@ public class SingleOutputStreamOperator<T> extends DataStream<T> {
     }
 
     /**
+     * Sets the slot sharing group of this operation. Parallel instances of operations that are in
+     * the same slot sharing group will be co-located in the same TaskManager slot, if possible.
+     *
+     * <p>Operations inherit the slot sharing group of input operations if all input operations are
+     * in the same slot sharing group and no slot sharing group was explicitly specified.
+     *
+     * <p>Initially an operation is in the default slot sharing group. An operation can be put into
+     * the default group explicitly by setting the slot sharing group with name {@code "default"}.
+     *
+     * @param slotSharingGroup which contains name and its resource spec.
+     */
+    @PublicEvolving
+    public SingleOutputStreamOperator<T> slotSharingGroup(SlotSharingGroup slotSharingGroup) {
+        transformation.setSlotSharingGroup(slotSharingGroup);
+        return this;
+    }
+
+    /**
      * Gets the {@link DataStream} that contains the elements that are emitted from an operation
      * into the side output with the given {@link OutputTag}.
      *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
index 5a1d26c..d39fd1f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
@@ -22,11 +22,14 @@ import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.RuntimeExecutionMode;
 import org.apache.flink.api.common.cache.DistributedCache;
+import org.apache.flink.api.common.operators.ResourceSpec;
+import org.apache.flink.api.common.operators.util.SlotSharingGroupUtils;
 import org.apache.flink.api.connector.source.Boundedness;
 import org.apache.flink.api.dag.Transformation;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.ExecutionOptions;
+import org.apache.flink.configuration.MemorySize;
 import org.apache.flink.configuration.ReadableConfig;
 import org.apache.flink.core.fs.Path;
 import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
@@ -141,7 +144,7 @@ public class StreamGraphGenerator {
 
     private final ReadableConfig configuration;
 
-    // Records the slot sharing groups and their corresponding ResourceProfile
+    // Records the slot sharing groups and their corresponding fine-grained ResourceProfile
     private final Map<String, ResourceProfile> slotSharingGroupResources = new HashMap<>();
 
     private Path savepointDir;
@@ -293,7 +296,12 @@ public class StreamGraphGenerator {
      */
     public StreamGraphGenerator setSlotSharingGroupResource(
             Map<String, ResourceProfile> slotSharingGroupResources) {
-        this.slotSharingGroupResources.putAll(slotSharingGroupResources);
+        slotSharingGroupResources.forEach(
+                (name, profile) -> {
+                    if (!profile.equals(ResourceProfile.UNKNOWN)) {
+                        this.slotSharingGroupResources.put(name, profile);
+                    }
+                });
         return this;
     }
 
@@ -312,6 +320,8 @@ public class StreamGraphGenerator {
             transform(transformation);
         }
 
+        streamGraph.setSlotSharingGroupResource(slotSharingGroupResources);
+
         for (StreamNode node : streamGraph.getStreamNodes()) {
             if (node.getInEdges().stream().anyMatch(this::shouldDisableUnalignedCheckpointing)) {
                 for (StreamEdge edge : node.getInEdges()) {
@@ -342,7 +352,6 @@ public class StreamGraphGenerator {
         graph.setTimeCharacteristic(timeCharacteristic);
         graph.setJobName(jobName);
         graph.setJobType(shouldExecuteInBatchMode ? JobType.BATCH : JobType.STREAMING);
-        graph.setSlotSharingGroupResource(slotSharingGroupResources);
 
         if (shouldExecuteInBatchMode) {
 
@@ -452,6 +461,33 @@ public class StreamGraphGenerator {
             }
         }
 
+        transform
+                .getSlotSharingGroup()
+                .ifPresent(
+                        slotSharingGroup -> {
+                            final ResourceSpec resourceSpec =
+                                    SlotSharingGroupUtils.extractResourceSpec(slotSharingGroup);
+                            if (!resourceSpec.equals(ResourceSpec.UNKNOWN)) {
+                                slotSharingGroupResources.compute(
+                                        slotSharingGroup.getName(),
+                                        (name, profile) -> {
+                                            if (profile == null) {
+                                                return ResourceProfile.fromResourceSpec(
+                                                        resourceSpec, MemorySize.ZERO);
+                                            } else if (!ResourceProfile.fromResourceSpec(
+                                                            resourceSpec, MemorySize.ZERO)
+                                                    .equals(profile)) {
+                                                throw new IllegalArgumentException(
+                                                        "The slot sharing group "
+                                                                + slotSharingGroup.getName()
+                                                                + " has been configured with two different resource spec.");
+                                            } else {
+                                                return profile;
+                                            }
+                                        });
+                            }
+                        });
+
         // call at least once to trigger exceptions about MissingTypeInfo
         transform.getOutputType();
 
@@ -720,7 +756,9 @@ public class StreamGraphGenerator {
 
         final String slotSharingGroup =
                 determineSlotSharingGroup(
-                        transform.getSlotSharingGroup(),
+                        transform.getSlotSharingGroup().isPresent()
+                                ? transform.getSlotSharingGroup().get().getName()
+                                : null,
                         allInputIds.stream()
                                 .flatMap(Collection::stream)
                                 .collect(Collectors.toList()));
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java
index 23e3807..4a59ec4 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.streaming.api.graph;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.api.common.operators.ResourceSpec;
+import org.apache.flink.api.common.operators.SlotSharingGroup;
 import org.apache.flink.api.common.state.MapStateDescriptor;
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
@@ -749,6 +750,62 @@ public class StreamGraphGeneratorTest extends TestLogger {
                 equalTo(SavepointRestoreSettings.forPath("/tmp/savepoint1")));
     }
 
+    @Test
+    public void testConfigureSlotSharingGroupResource() {
+        final SlotSharingGroup ssg1 =
+                SlotSharingGroup.newBuilder("ssg1").setCpuCores(1).setTaskHeapMemoryMB(100).build();
+        final SlotSharingGroup ssg2 =
+                SlotSharingGroup.newBuilder("ssg2").setCpuCores(2).setTaskHeapMemoryMB(200).build();
+        final SlotSharingGroup ssg3 =
+                SlotSharingGroup.newBuilder(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP)
+                        .setCpuCores(3)
+                        .setTaskHeapMemoryMB(300)
+                        .build();
+        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+        final DataStream<Integer> source = env.fromElements(1).slotSharingGroup("ssg1");
+        source.map(value -> value)
+                .slotSharingGroup(ssg2)
+                .map(value -> value * 2)
+                .map(value -> value * 3)
+                .slotSharingGroup(SlotSharingGroup.newBuilder("ssg4").build())
+                .map(value -> value * 4)
+                .slotSharingGroup(ssg3)
+                .addSink(new DiscardingSink<>())
+                .slotSharingGroup(ssg1);
+
+        final StreamGraph streamGraph = env.getStreamGraph();
+        assertThat(
+                streamGraph.getSlotSharingGroupResource("ssg1").get(),
+                is(ResourceProfile.fromResources(1, 100)));
+        assertThat(
+                streamGraph.getSlotSharingGroupResource("ssg2").get(),
+                is(ResourceProfile.fromResources(2, 200)));
+        assertThat(
+                streamGraph
+                        .getSlotSharingGroupResource(
+                                StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP)
+                        .get(),
+                is(ResourceProfile.fromResources(3, 300)));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testConflictSlotSharingGroup() {
+        final SlotSharingGroup ssg =
+                SlotSharingGroup.newBuilder("ssg").setCpuCores(1).setTaskHeapMemoryMB(100).build();
+        final SlotSharingGroup ssgConflict =
+                SlotSharingGroup.newBuilder("ssg").setCpuCores(2).setTaskHeapMemoryMB(200).build();
+        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+        final DataStream<Integer> source = env.fromElements(1).slotSharingGroup(ssg);
+        source.map(value -> value)
+                .slotSharingGroup(ssgConflict)
+                .addSink(new DiscardingSink<>())
+                .slotSharingGroup(ssgConflict);
+
+        env.getStreamGraph();
+    }
+
     private static class OutputTypeConfigurableFunction<T>
             implements OutputTypeConfigurable<T>, Function {
         private TypeInformation<T> typeInformation;

[flink] 07/07: [FLINK-21925][core] Introduce fine-grained.shuffle-mode.all-blocking to avoid resource deadlock in batch jobs that apply fine-grained resource management

Posted by xt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

xtsong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 25fdf5490db301b33157c99fe830d3497b6d0bdd
Author: Yangze Guo <ka...@gmail.com>
AuthorDate: Mon Jun 28 11:07:52 2021 +0800

    [FLINK-21925][core] Introduce fine-grained.shuffle-mode.all-blocking to avoid resource deadlock in batch jobs that apply fine-grained resource management
    
    This closes #16307
---
 .../apache/flink/configuration/ClusterOptions.java  |  8 ++++++++
 .../api/environment/StreamExecutionEnvironment.java | 21 +++++++++++++++++++++
 .../flink/streaming/api/graph/StreamGraph.java      |  5 +++++
 3 files changed, 34 insertions(+)

diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ClusterOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/ClusterOptions.java
index 1bb007b..667dc76 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ClusterOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ClusterOptions.java
@@ -123,6 +123,14 @@ public class ClusterOptions {
                     .withDescription(
                             "Defines whether the cluster uses fine-grained resource management.");
 
+    @Documentation.ExcludeFromDocumentation
+    public static final ConfigOption<Boolean> FINE_GRAINED_SHUFFLE_MODE_ALL_BLOCKING =
+            ConfigOptions.key("fine-grained.shuffle-mode.all-blocking")
+                    .booleanType()
+                    .defaultValue(false)
+                    .withDescription(
+                            "Whether to convert all PIPELINE edges to BLOCKING when apply fine-grained resource management in batch jobs.");
+
     public static JobManagerOptions.SchedulerType getSchedulerType(Configuration configuration) {
         if (isAdaptiveSchedulerEnabled(configuration) || isReactiveModeEnabled(configuration)) {
             return JobManagerOptions.SchedulerType.Adaptive;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index ba5d107..9670b39 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -52,10 +52,12 @@ import org.apache.flink.api.java.typeutils.PojoTypeInfo;
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
 import org.apache.flink.configuration.CheckpointingOptions;
+import org.apache.flink.configuration.ClusterOptions;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.CoreOptions;
 import org.apache.flink.configuration.DeploymentOptions;
 import org.apache.flink.configuration.ExecutionOptions;
+import org.apache.flink.configuration.IllegalConfigurationException;
 import org.apache.flink.configuration.MemorySize;
 import org.apache.flink.configuration.PipelineOptions;
 import org.apache.flink.configuration.ReadableConfig;
@@ -91,6 +93,7 @@ import org.apache.flink.streaming.api.functions.source.SocketTextStreamFunction;
 import org.apache.flink.streaming.api.functions.source.SourceFunction;
 import org.apache.flink.streaming.api.functions.source.StatefulSequenceSource;
 import org.apache.flink.streaming.api.functions.source.TimestampedFileInputSplit;
+import org.apache.flink.streaming.api.graph.GlobalDataExchangeMode;
 import org.apache.flink.streaming.api.graph.StreamGraph;
 import org.apache.flink.streaming.api.graph.StreamGraphGenerator;
 import org.apache.flink.streaming.api.graph.StreamingJobGraphGenerator;
@@ -2099,6 +2102,24 @@ public class StreamExecutionEnvironment {
     @Internal
     public StreamGraph getStreamGraph(String jobName, boolean clearTransformations) {
         StreamGraph streamGraph = getStreamGraphGenerator().setJobName(jobName).generate();
+
+        // There might be a resource deadlock when applying fine-grained resource management in
+        // batch jobs with PIPELINE edges. Users need to trigger the
+        // fine-grained.shuffle-mode.all-blocking to convert all edges to BLOCKING before we fix
+        // that issue.
+        if (configuration.get(ExecutionOptions.RUNTIME_MODE) == RuntimeExecutionMode.BATCH
+                && streamGraph.hasFineGrainedResource()) {
+            if (configuration.get(ClusterOptions.FINE_GRAINED_SHUFFLE_MODE_ALL_BLOCKING)) {
+                streamGraph.setGlobalDataExchangeMode(GlobalDataExchangeMode.ALL_EDGES_BLOCKING);
+            } else {
+                throw new IllegalConfigurationException(
+                        "At the moment, fine-grained resource management requires batch workloads to "
+                                + "be executed with types of all edges being BLOCKING. To do that, you need to configure '"
+                                + ClusterOptions.FINE_GRAINED_SHUFFLE_MODE_ALL_BLOCKING.key()
+                                + "' to 'true'. Notice that this may affect the performance. See FLINK-20865 for more details.");
+            }
+        }
+
         if (clearTransformations) {
             this.transformations.clear();
         }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
index 35f8b45..e3f284f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
@@ -253,6 +253,11 @@ public class StreamGraph implements Pipeline {
         return Optional.ofNullable(slotSharingGroupResources.get(groupId));
     }
 
+    public boolean hasFineGrainedResource() {
+        return slotSharingGroupResources.values().stream()
+                .anyMatch(resourceProfile -> !resourceProfile.equals(ResourceProfile.UNKNOWN));
+    }
+
     /**
      * Set whether to put all vertices into the same slot sharing group by default.
      *

[flink] 03/07: [FLINK-21925][core] Introduce public SlotSharingGroup

Posted by xt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

xtsong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 0544ddd0e643abced35c353f8810db01f2da5090
Author: Yangze Guo <ka...@gmail.com>
AuthorDate: Wed Jun 9 11:01:20 2021 +0800

    [FLINK-21925][core] Introduce public SlotSharingGroup
    
    Users can use this class to describe the name and the the different resource components of a slot sharing group.
---
 .../api/common/operators/SlotSharingGroup.java     | 235 +++++++++++++++++++++
 .../api/common/operators/SlotSharingGroupTest.java |  84 ++++++++
 2 files changed, 319 insertions(+)

diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java
new file mode 100644
index 0000000..029788f
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.operators;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.resources.CPUResource;
+import org.apache.flink.configuration.MemorySize;
+
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/** Describe the name and the the different resource components of a slot sharing group. */
+@PublicEvolving
+public class SlotSharingGroup implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    private final String name;
+
+    /** How many cpu cores are needed. Can be null only if it is unknown. */
+    @Nullable // can be null only for UNKNOWN
+    private final CPUResource cpuCores;
+
+    /** How much task heap memory is needed. */
+    @Nullable // can be null only for UNKNOWN
+    private final MemorySize taskHeapMemory;
+
+    /** How much task off-heap memory is needed. */
+    @Nullable // can be null only for UNKNOWN
+    private final MemorySize taskOffHeapMemory;
+
+    /** How much managed memory is needed. */
+    @Nullable // can be null only for UNKNOWN
+    private final MemorySize managedMemory;
+
+    /** A extensible field for user specified resources from {@link SlotSharingGroup}. */
+    private final Map<String, Double> externalResources = new HashMap<>();
+
+    private SlotSharingGroup(
+            String name,
+            CPUResource cpuCores,
+            MemorySize taskHeapMemory,
+            MemorySize taskOffHeapMemory,
+            MemorySize managedMemory,
+            Map<String, Double> extendedResources) {
+        this.name = checkNotNull(name);
+        this.cpuCores = checkNotNull(cpuCores);
+        this.taskHeapMemory = checkNotNull(taskHeapMemory);
+        this.taskOffHeapMemory = checkNotNull(taskOffHeapMemory);
+        this.managedMemory = checkNotNull(managedMemory);
+        this.externalResources.putAll(checkNotNull(extendedResources));
+    }
+
+    private SlotSharingGroup(String name) {
+        this.name = checkNotNull(name);
+        this.cpuCores = null;
+        this.taskHeapMemory = null;
+        this.taskOffHeapMemory = null;
+        this.managedMemory = null;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public Optional<MemorySize> getManagedMemory() {
+        return Optional.ofNullable(managedMemory);
+    }
+
+    public Optional<MemorySize> getTaskHeapMemory() {
+        return Optional.ofNullable(taskHeapMemory);
+    }
+
+    public Optional<MemorySize> getTaskOffHeapMemory() {
+        return Optional.ofNullable(taskOffHeapMemory);
+    }
+
+    public Optional<Double> getCpuCores() {
+        return Optional.ofNullable(cpuCores)
+                .map(cpuResource -> cpuResource.getValue().doubleValue());
+    }
+
+    public Map<String, Double> getExternalResources() {
+        return Collections.unmodifiableMap(externalResources);
+    }
+
+    public static Builder newBuilder(String name) {
+        return new Builder(name);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == this) {
+            return true;
+        } else if (obj != null && obj.getClass() == SlotSharingGroup.class) {
+            SlotSharingGroup that = (SlotSharingGroup) obj;
+            return Objects.equals(this.cpuCores, that.cpuCores)
+                    && Objects.equals(taskHeapMemory, that.taskHeapMemory)
+                    && Objects.equals(taskOffHeapMemory, that.taskOffHeapMemory)
+                    && Objects.equals(managedMemory, that.managedMemory)
+                    && Objects.equals(externalResources, that.externalResources);
+        }
+        return false;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = Objects.hashCode(cpuCores);
+        result = 31 * result + Objects.hashCode(taskHeapMemory);
+        result = 31 * result + Objects.hashCode(taskOffHeapMemory);
+        result = 31 * result + Objects.hashCode(managedMemory);
+        result = 31 * result + externalResources.hashCode();
+        return result;
+    }
+
+    /** Builder for the {@link SlotSharingGroup}. */
+    public static class Builder {
+
+        private String name;
+        private CPUResource cpuCores;
+        private MemorySize taskHeapMemory;
+        private MemorySize taskOffHeapMemory;
+        private MemorySize managedMemory;
+        private Map<String, Double> externalResources = new HashMap<>();
+
+        private Builder(String name) {
+            this.name = name;
+        }
+
+        /** Set the CPU cores for this SlotSharingGroup. */
+        public Builder setCpuCores(double cpuCores) {
+            checkArgument(cpuCores > 0, "The cpu cores should be positive.");
+            this.cpuCores = new CPUResource(cpuCores);
+            return this;
+        }
+
+        /** Set the task heap memory for this SlotSharingGroup. */
+        public Builder setTaskHeapMemory(MemorySize taskHeapMemory) {
+            checkArgument(
+                    taskHeapMemory.compareTo(MemorySize.ZERO) > 0,
+                    "The task heap memory should be positive.");
+            this.taskHeapMemory = taskHeapMemory;
+            return this;
+        }
+
+        /** Set the task heap memory for this SlotSharingGroup in MB. */
+        public Builder setTaskHeapMemoryMB(int taskHeapMemoryMB) {
+            checkArgument(taskHeapMemoryMB > 0, "The task heap memory should be positive.");
+            this.taskHeapMemory = MemorySize.ofMebiBytes(taskHeapMemoryMB);
+            return this;
+        }
+
+        /** Set the task off-heap memory for this SlotSharingGroup. */
+        public Builder setTaskOffHeapMemory(MemorySize taskOffHeapMemory) {
+            this.taskOffHeapMemory = taskOffHeapMemory;
+            return this;
+        }
+
+        /** Set the task off-heap memory for this SlotSharingGroup in MB. */
+        public Builder setTaskOffHeapMemoryMB(int taskOffHeapMemoryMB) {
+            this.taskOffHeapMemory = MemorySize.ofMebiBytes(taskOffHeapMemoryMB);
+            return this;
+        }
+
+        /** Set the task managed memory for this SlotSharingGroup. */
+        public Builder setManagedMemory(MemorySize managedMemory) {
+            this.managedMemory = managedMemory;
+            return this;
+        }
+
+        /** Set the task managed memory for this SlotSharingGroup in MB. */
+        public Builder setManagedMemoryMB(int managedMemoryMB) {
+            this.managedMemory = MemorySize.ofMebiBytes(managedMemoryMB);
+            return this;
+        }
+
+        /**
+         * Add the given external resource. The old value with the same resource name will be
+         * replaced if present.
+         */
+        public Builder setExternalResource(String name, double value) {
+            this.externalResources.put(name, value);
+            return this;
+        }
+
+        /** Build the SlotSharingGroup. */
+        public SlotSharingGroup build() {
+            if (cpuCores != null && taskHeapMemory != null) {
+                taskOffHeapMemory = Optional.ofNullable(taskOffHeapMemory).orElse(MemorySize.ZERO);
+                managedMemory = Optional.ofNullable(managedMemory).orElse(MemorySize.ZERO);
+                return new SlotSharingGroup(
+                        name,
+                        cpuCores,
+                        taskHeapMemory,
+                        taskOffHeapMemory,
+                        managedMemory,
+                        externalResources);
+            } else if (cpuCores != null
+                    || taskHeapMemory != null
+                    || taskOffHeapMemory != null
+                    || managedMemory != null
+                    || !externalResources.isEmpty()) {
+                throw new IllegalArgumentException(
+                        "The cpu cores and task heap memory are required when specifying the resource of a slot sharing group. "
+                                + "You need to explicitly configure them with positive value.");
+            } else {
+                return new SlotSharingGroup(name);
+            }
+        }
+    }
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/operators/SlotSharingGroupTest.java b/flink-core/src/test/java/org/apache/flink/api/common/operators/SlotSharingGroupTest.java
new file mode 100644
index 0000000..e48920d
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/common/operators/SlotSharingGroupTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.operators;
+
+import org.apache.flink.configuration.MemorySize;
+
+import org.junit.Test;
+
+import java.util.Collections;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/** Tests for {@link SlotSharingGroup}. */
+public class SlotSharingGroupTest {
+    @Test
+    public void testBuildSlotSharingGroupWithSpecificResource() {
+        final String name = "ssg";
+        final MemorySize heap = MemorySize.ofMebiBytes(100);
+        final MemorySize offHeap = MemorySize.ofMebiBytes(200);
+        final MemorySize managed = MemorySize.ofMebiBytes(300);
+        final SlotSharingGroup slotSharingGroup =
+                SlotSharingGroup.newBuilder(name)
+                        .setCpuCores(1)
+                        .setTaskHeapMemory(heap)
+                        .setTaskOffHeapMemory(offHeap)
+                        .setManagedMemory(managed)
+                        .setExternalResource("gpu", 1)
+                        .build();
+
+        assertThat(slotSharingGroup.getName(), is(name));
+        assertThat(slotSharingGroup.getCpuCores().get(), is(1.0));
+        assertThat(slotSharingGroup.getTaskHeapMemory().get(), is(heap));
+        assertThat(slotSharingGroup.getTaskOffHeapMemory().get(), is(offHeap));
+        assertThat(slotSharingGroup.getManagedMemory().get(), is(managed));
+        assertThat(
+                slotSharingGroup.getExternalResources(), is(Collections.singletonMap("gpu", 1.0)));
+    }
+
+    @Test
+    public void testBuildSlotSharingGroupWithUnknownResource() {
+        final String name = "ssg";
+        final SlotSharingGroup slotSharingGroup = SlotSharingGroup.newBuilder(name).build();
+
+        assertThat(slotSharingGroup.getName(), is(name));
+        assertFalse(slotSharingGroup.getCpuCores().isPresent());
+        assertFalse(slotSharingGroup.getTaskHeapMemory().isPresent());
+        assertFalse(slotSharingGroup.getManagedMemory().isPresent());
+        assertFalse(slotSharingGroup.getTaskOffHeapMemory().isPresent());
+        assertTrue(slotSharingGroup.getExternalResources().isEmpty());
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testBuildSlotSharingGroupWithIllegalConfig() {
+        SlotSharingGroup.newBuilder("ssg")
+                .setCpuCores(1)
+                .setTaskHeapMemory(MemorySize.ZERO)
+                .setTaskOffHeapMemoryMB(10)
+                .build();
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testBuildSlotSharingGroupWithoutAllRequiredConfig() {
+        SlotSharingGroup.newBuilder("ssg").setCpuCores(1).setTaskOffHeapMemoryMB(10).build();
+    }
+}

[flink] 02/07: [hotfix][core] Check the cpu and heap memory to be positive when building ResourceSpec

Posted by xt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

xtsong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit bc5f36d9fb063a5af1264ae58bbea9f7c0d54f3b
Author: Yangze Guo <ka...@gmail.com>
AuthorDate: Fri Jul 2 15:15:50 2021 +0800

    [hotfix][core] Check the cpu and heap memory to be positive when building ResourceSpec
---
 .../org/apache/flink/api/common/operators/ResourceSpec.java   | 11 ++++++++++-
 .../org/apache/flink/runtime/dispatcher/DispatcherTest.java   |  2 +-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/ResourceSpec.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/ResourceSpec.java
index f66f2a3..6b24b66 100755
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/ResourceSpec.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/ResourceSpec.java
@@ -26,6 +26,7 @@ import org.apache.flink.configuration.MemorySize;
 import javax.annotation.Nullable;
 
 import java.io.Serializable;
+import java.math.BigDecimal;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -70,7 +71,13 @@ public final class ResourceSpec implements Serializable {
     public static final ResourceSpec DEFAULT = UNKNOWN;
 
     /** A ResourceSpec that indicates zero amount of resources. */
-    public static final ResourceSpec ZERO = ResourceSpec.newBuilder(0.0, 0).build();
+    public static final ResourceSpec ZERO =
+            new ResourceSpec(
+                    new CPUResource(0.0),
+                    MemorySize.ZERO,
+                    MemorySize.ZERO,
+                    MemorySize.ZERO,
+                    Collections.emptyMap());
 
     /** How many cpu cores are needed. Can be null only if it is unknown. */
     @Nullable private final CPUResource cpuCores;
@@ -397,6 +404,8 @@ public final class ResourceSpec implements Serializable {
         }
 
         public ResourceSpec build() {
+            checkArgument(cpuCores.getValue().compareTo(BigDecimal.ZERO) > 0);
+            checkArgument(taskHeapMemory.compareTo(MemorySize.ZERO) > 0);
             return new ResourceSpec(
                     cpuCores, taskHeapMemory, taskOffHeapMemory, managedMemory, extendedResources);
         }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java
index 4cc63e0..3b7b210 100755
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java
@@ -269,7 +269,7 @@ public class DispatcherTest extends TestLogger {
      */
     @Test
     public void testJobSubmissionWithPartialResourceConfigured() throws Exception {
-        ResourceSpec resourceSpec = ResourceSpec.newBuilder(2.0, 0).build();
+        ResourceSpec resourceSpec = ResourceSpec.newBuilder(2.0, 10).build();
 
         final JobVertex firstVertex = new JobVertex("firstVertex");
         firstVertex.setInvokableClass(NoOpInvokable.class);

[flink] 06/07: [FLINK-21925][core] Introduce StreamExecutionEnvironment#registerSlotSharingGroup

Posted by xt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

xtsong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 3f36bbf79ce9b34228f20f2bba50c85d1b6a29e3
Author: Yangze Guo <ka...@gmail.com>
AuthorDate: Fri Jun 25 15:50:06 2021 +0800

    [FLINK-21925][core] Introduce StreamExecutionEnvironment#registerSlotSharingGroup
---
 ...st_stream_execution_environment_completeness.py |  3 +-
 .../environment/StreamExecutionEnvironment.java    | 37 +++++++++++++++++-
 .../api/StreamExecutionEnvironmentTest.java        | 44 ++++++++++++++++++++++
 .../api/scala/StreamExecutionEnvironment.scala     | 17 +++++++++
 4 files changed, 99 insertions(+), 2 deletions(-)

diff --git a/flink-python/pyflink/datastream/tests/test_stream_execution_environment_completeness.py b/flink-python/pyflink/datastream/tests/test_stream_execution_environment_completeness.py
index 878d042..5f6a1a8 100644
--- a/flink-python/pyflink/datastream/tests/test_stream_execution_environment_completeness.py
+++ b/flink-python/pyflink/datastream/tests/test_stream_execution_environment_completeness.py
@@ -39,6 +39,7 @@ class StreamExecutionEnvironmentCompletenessTests(PythonAPICompletenessTestCase,
         # Currently only the methods for configuration is added.
         # 'isForceCheckpointing', 'getNumberOfExecutionRetries', 'setNumberOfExecutionRetries'
         # is deprecated, exclude them.
+        # TODO the registerSlotSharingGroup should be removed from this list after FLINK-23165.
         return {'getLastJobExecutionResult', 'getId', 'getIdString',
                 'registerCachedFile', 'createCollectionsEnvironment', 'createLocalEnvironment',
                 'createRemoteEnvironment', 'addOperator', 'fromElements',
@@ -48,7 +49,7 @@ class StreamExecutionEnvironmentCompletenessTests(PythonAPICompletenessTestCase,
                 'createInput', 'createLocalEnvironmentWithWebUI', 'fromCollection',
                 'socketTextStream', 'initializeContextEnvironment', 'readTextFile',
                 'setNumberOfExecutionRetries', 'configure', 'executeAsync', 'registerJobListener',
-                'clearJobListeners', 'getJobListeners', "fromSequence"}
+                'clearJobListeners', 'getJobListeners', "fromSequence", "registerSlotSharingGroup"}
 
 
 if __name__ == '__main__':
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 77efe15..ba5d107 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -31,6 +31,9 @@ import org.apache.flink.api.common.functions.InvalidTypesException;
 import org.apache.flink.api.common.io.FileInputFormat;
 import org.apache.flink.api.common.io.FilePathFilter;
 import org.apache.flink.api.common.io.InputFormat;
+import org.apache.flink.api.common.operators.ResourceSpec;
+import org.apache.flink.api.common.operators.SlotSharingGroup;
+import org.apache.flink.api.common.operators.util.SlotSharingGroupUtils;
 import org.apache.flink.api.common.restartstrategy.RestartStrategies;
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
@@ -53,6 +56,7 @@ import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.CoreOptions;
 import org.apache.flink.configuration.DeploymentOptions;
 import org.apache.flink.configuration.ExecutionOptions;
+import org.apache.flink.configuration.MemorySize;
 import org.apache.flink.configuration.PipelineOptions;
 import org.apache.flink.configuration.ReadableConfig;
 import org.apache.flink.configuration.RestOptions;
@@ -64,6 +68,7 @@ import org.apache.flink.core.execution.PipelineExecutor;
 import org.apache.flink.core.execution.PipelineExecutorFactory;
 import org.apache.flink.core.execution.PipelineExecutorServiceLoader;
 import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
 import org.apache.flink.runtime.state.KeyGroupRangeAssignment;
 import org.apache.flink.runtime.state.StateBackend;
 import org.apache.flink.runtime.state.StateBackendLoader;
@@ -110,8 +115,10 @@ import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
@@ -189,6 +196,9 @@ public class StreamExecutionEnvironment {
 
     private final List<JobListener> jobListeners = new ArrayList<>();
 
+    // Records the slot sharing groups and their corresponding fine-grained ResourceProfile
+    private final Map<String, ResourceProfile> slotSharingGroupResources = new HashMap<>();
+
     // --------------------------------------------------------------------------------------------
     // Constructor and Properties
     // --------------------------------------------------------------------------------------------
@@ -337,6 +347,30 @@ public class StreamExecutionEnvironment {
     }
 
     /**
+     * Register a slot sharing group with its resource spec.
+     *
+     * <p>Note that a slot sharing group hints the scheduler that the grouped operators CAN be
+     * deployed into a shared slot. There's no guarantee that the scheduler always deploy the
+     * grouped operators together. In cases grouped operators are deployed into separate slots, the
+     * slot resources will be derived from the specified group requirements.
+     *
+     * @param slotSharingGroup which contains name and its resource spec.
+     */
+    @PublicEvolving
+    public StreamExecutionEnvironment registerSlotSharingGroup(SlotSharingGroup slotSharingGroup) {
+        final ResourceSpec resourceSpec =
+                SlotSharingGroupUtils.extractResourceSpec(slotSharingGroup);
+        if (!resourceSpec.equals(ResourceSpec.UNKNOWN)) {
+            this.slotSharingGroupResources.put(
+                    slotSharingGroup.getName(),
+                    ResourceProfile.fromResourceSpec(
+                            SlotSharingGroupUtils.extractResourceSpec(slotSharingGroup),
+                            MemorySize.ZERO));
+        }
+        return this;
+    }
+
+    /**
      * Gets the parallelism with which operation are executed by default. Operations can
      * individually override this value to use a specific parallelism.
      *
@@ -2087,7 +2121,8 @@ public class StreamExecutionEnvironment {
                 .setChaining(isChainingEnabled)
                 .setUserArtifacts(cacheFile)
                 .setTimeCharacteristic(timeCharacteristic)
-                .setDefaultBufferTimeout(bufferTimeout);
+                .setDefaultBufferTimeout(bufferTimeout)
+                .setSlotSharingGroupResource(slotSharingGroupResources);
     }
 
     /**
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java
index f86dad2..683e31f 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java
@@ -18,6 +18,7 @@
 package org.apache.flink.streaming.api;
 
 import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.operators.SlotSharingGroup;
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.common.typeinfo.Types;
@@ -25,6 +26,7 @@ import org.apache.flink.api.java.typeutils.GenericTypeInfo;
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.datastream.DataStreamSource;
 import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
@@ -48,7 +50,10 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.NoSuchElementException;
 
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -275,6 +280,45 @@ public class StreamExecutionEnvironmentTest {
     }
 
     @Test
+    public void testRegisterSlotSharingGroup() {
+        final SlotSharingGroup ssg1 =
+                SlotSharingGroup.newBuilder("ssg1").setCpuCores(1).setTaskHeapMemoryMB(100).build();
+        final SlotSharingGroup ssg2 =
+                SlotSharingGroup.newBuilder("ssg2").setCpuCores(2).setTaskHeapMemoryMB(200).build();
+        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+        env.registerSlotSharingGroup(ssg1);
+        env.registerSlotSharingGroup(ssg2);
+        env.registerSlotSharingGroup(SlotSharingGroup.newBuilder("ssg3").build());
+
+        final DataStream<Integer> source = env.fromElements(1).slotSharingGroup("ssg1");
+        source.map(value -> value).slotSharingGroup(ssg2).addSink(new DiscardingSink<>());
+
+        final StreamGraph streamGraph = env.getStreamGraph();
+        assertThat(
+                streamGraph.getSlotSharingGroupResource("ssg1").get(),
+                is(ResourceProfile.fromResources(1, 100)));
+        assertThat(
+                streamGraph.getSlotSharingGroupResource("ssg2").get(),
+                is(ResourceProfile.fromResources(2, 200)));
+        assertFalse(streamGraph.getSlotSharingGroupResource("ssg3").isPresent());
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testRegisterSlotSharingGroupConflict() {
+        final SlotSharingGroup ssg =
+                SlotSharingGroup.newBuilder("ssg1").setCpuCores(1).setTaskHeapMemoryMB(100).build();
+        final SlotSharingGroup ssgConflict =
+                SlotSharingGroup.newBuilder("ssg1").setCpuCores(2).setTaskHeapMemoryMB(200).build();
+        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+        env.registerSlotSharingGroup(ssg);
+
+        final DataStream<Integer> source = env.fromElements(1).slotSharingGroup("ssg1");
+        source.map(value -> value).slotSharingGroup(ssgConflict).addSink(new DiscardingSink<>());
+
+        env.getStreamGraph();
+    }
+
+    @Test
     public void testGetStreamGraph() {
         try {
             TypeInformation<Integer> typeInfo = BasicTypeInfo.INT_TYPE_INFO;
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
index 3ee460c..d0551c3 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
@@ -24,6 +24,7 @@ import org.apache.flink.annotation.{Experimental, Internal, Public, PublicEvolvi
 import org.apache.flink.api.common.RuntimeExecutionMode
 import org.apache.flink.api.common.eventtime.WatermarkStrategy
 import org.apache.flink.api.common.io.{FileInputFormat, FilePathFilter, InputFormat}
+import org.apache.flink.api.common.operators.SlotSharingGroup
 import org.apache.flink.api.common.restartstrategy.RestartStrategies.RestartStrategyConfiguration
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.connector.source.{Source, SourceSplit}
@@ -107,6 +108,22 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
   }
 
   /**
+   * Register a slot sharing group with its resource spec.
+   *
+   * <p>Note that a slot sharing group hints the scheduler that the grouped operators CAN be
+   * deployed into a shared slot. There's no guarantee that the scheduler always deploy the
+   * grouped operators together. In cases grouped operators are deployed into separate slots, the
+   * slot resources will be derived from the specified group requirements.
+   *
+   * @param slotSharingGroup which contains name and its resource spec.
+   */
+  @PublicEvolving
+  def registerSlotSharingGroup(slotSharingGroup: SlotSharingGroup): StreamExecutionEnvironment = {
+    javaEnv.registerSlotSharingGroup(slotSharingGroup)
+    this
+  }
+
+  /**
    * Returns the default parallelism for this execution environment. Note that this
    * value can be overridden by individual operations using [[DataStream#setParallelism(int)]]
    */

[flink] 01/07: [hotfix][core] Fix typo in the Builder of ResourceSpec

Posted by xt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

xtsong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit d0846bbc8de210eb84cd1dd7469b27cf2f319f95
Author: Yangze Guo <ka...@gmail.com>
AuthorDate: Wed Jun 30 19:21:56 2021 +0800

    [hotfix][core] Fix typo in the Builder of ResourceSpec
---
 .../main/java/org/apache/flink/api/common/operators/ResourceSpec.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/ResourceSpec.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/ResourceSpec.java
index 218a70f..f66f2a3 100755
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/ResourceSpec.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/ResourceSpec.java
@@ -359,7 +359,7 @@ public final class ResourceSpec implements Serializable {
             return this;
         }
 
-        public Builder setOffTaskHeapMemoryMB(int taskOffHeapMemoryMB) {
+        public Builder setTaskOffHeapMemoryMB(int taskOffHeapMemoryMB) {
             this.taskOffHeapMemory = MemorySize.ofMebiBytes(taskOffHeapMemoryMB);
             return this;
         }