You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gu...@apache.org on 2024/05/22 04:04:37 UTC

(flink) 02/11: [FLINK-34549][API] Introduce SlotSharingGroup for v2 and move MemorySize to flink-core-api

This is an automated email from the ASF dual-hosted git repository.

guoweijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 1d73510bc2e30d5352a8cbbafd7939197a32cedb
Author: Weijie Guo <re...@163.com>
AuthorDate: Tue Apr 16 14:42:48 2024 +0800

    [FLINK-34549][API] Introduce SlotSharingGroup for v2 and move MemorySize to flink-core-api
---
 .../5b9eed8a-5fb6-4373-98ac-3be2a71941b8           |   1 +
 flink-core-api/pom.xml                             |   7 ++
 .../apache/flink/api/common}/SlotSharingGroup.java | 102 +++++++++++----------
 .../org/apache/flink/configuration/MemorySize.java |  28 ++++--
 .../flink/api/common/SlotSharingGroupTest.java     |  91 ++++++++++++++++++
 .../MemorySizePrettyPrintingTest.java              |   0
 .../apache/flink/configuration/MemorySizeTest.java |   0
 .../api/common/operators/SlotSharingGroup.java     |  31 +++++++
 flink-table/flink-sql-jdbc-driver-bundle/pom.xml   |   1 +
 9 files changed, 202 insertions(+), 59 deletions(-)

diff --git a/flink-architecture-tests/flink-architecture-tests-production/archunit-violations/5b9eed8a-5fb6-4373-98ac-3be2a71941b8 b/flink-architecture-tests/flink-architecture-tests-production/archunit-violations/5b9eed8a-5fb6-4373-98ac-3be2a71941b8
index 2c83601f889..a40e8acda25 100644
--- a/flink-architecture-tests/flink-architecture-tests-production/archunit-violations/5b9eed8a-5fb6-4373-98ac-3be2a71941b8
+++ b/flink-architecture-tests/flink-architecture-tests-production/archunit-violations/5b9eed8a-5fb6-4373-98ac-3be2a71941b8
@@ -3,6 +3,7 @@ org.apache.flink.api.common.eventtime.WatermarkGeneratorSupplier.createWatermark
 org.apache.flink.api.common.operators.Operator.getMinResources(): Returned leaf type org.apache.flink.api.common.operators.ResourceSpec does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
 org.apache.flink.api.common.operators.Operator.getPreferredResources(): Returned leaf type org.apache.flink.api.common.operators.ResourceSpec does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
 org.apache.flink.api.common.operators.Operator.setResources(org.apache.flink.api.common.operators.ResourceSpec, org.apache.flink.api.common.operators.ResourceSpec): Argument leaf type org.apache.flink.api.common.operators.ResourceSpec does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
+org.apache.flink.api.common.operators.SlotSharingGroup.from(org.apache.flink.api.common.SlotSharingGroup): Argument leaf type org.apache.flink.api.common.SlotSharingGroup does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
 org.apache.flink.api.common.operators.SlotSharingGroup.newBuilder(java.lang.String): Returned leaf type org.apache.flink.api.common.operators.SlotSharingGroup$Builder does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
 org.apache.flink.api.common.state.AggregatingStateDescriptor.getType(): Returned leaf type org.apache.flink.api.common.state.StateDescriptor$Type does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
 org.apache.flink.api.common.state.ListStateDescriptor.getType(): Returned leaf type org.apache.flink.api.common.state.StateDescriptor$Type does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
diff --git a/flink-core-api/pom.xml b/flink-core-api/pom.xml
index 8b95bc57e8e..a66ca8b386a 100644
--- a/flink-core-api/pom.xml
+++ b/flink-core-api/pom.xml
@@ -39,5 +39,12 @@ under the License.
 			<artifactId>flink-annotations</artifactId>
 			<version>${project.version}</version>
 		</dependency>
+
+		<!-- ================== test dependencies ================== -->
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-test-utils-junit</artifactId>
+		</dependency>
 	</dependencies>
 </project>
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java b/flink-core-api/src/main/java/org/apache/flink/api/common/SlotSharingGroup.java
similarity index 72%
copy from flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java
copy to flink-core-api/src/main/java/org/apache/flink/api/common/SlotSharingGroup.java
index 7a24dcfde57..fc8268cf9e7 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java
+++ b/flink-core-api/src/main/java/org/apache/flink/api/common/SlotSharingGroup.java
@@ -16,34 +16,33 @@
  * limitations under the License.
  */
 
-package org.apache.flink.api.common.operators;
+package org.apache.flink.api.common;
 
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.resources.CPUResource;
+import org.apache.flink.annotation.Experimental;
 import org.apache.flink.configuration.MemorySize;
 
 import javax.annotation.Nullable;
 
-import java.io.Serializable;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/** Describe the name and the different resource components of a slot sharing group. */
-@PublicEvolving
-public class SlotSharingGroup implements Serializable {
-    private static final long serialVersionUID = 1L;
-
+/**
+ * Describe the name and the different resource components of a slot sharing group.
+ *
+ * <p>Two SlotSharingGroup classes currently exist in flink-core and flink-core-api, because the one
+ * in flink-core exposes components marked as internal, which we want to avoid in DataStream-V2 API.
+ * Once the V1 API is removed, we will remove the legacy SlotSharingGroup.
+ */
+@Experimental
+public class SlotSharingGroup {
     private final String name;
 
     /** How many cpu cores are needed. Can be null only if it is unknown. */
     @Nullable // can be null only for UNKNOWN
-    private final CPUResource cpuCores;
+    private final Double cpuCores;
 
     /** How much task heap memory is needed. */
     @Nullable // can be null only for UNKNOWN
@@ -58,54 +57,53 @@ public class SlotSharingGroup implements Serializable {
     private final MemorySize managedMemory;
 
     /** A extensible field for user specified resources from {@link SlotSharingGroup}. */
-    private final Map<String, Double> externalResources = new HashMap<>();
+    private final Map<String, Double> externalResources;
 
     private SlotSharingGroup(
             String name,
-            CPUResource cpuCores,
-            MemorySize taskHeapMemory,
-            MemorySize taskOffHeapMemory,
-            MemorySize managedMemory,
+            @Nullable Double cpuCores,
+            @Nullable MemorySize taskHeapMemory,
+            @Nullable MemorySize taskOffHeapMemory,
+            @Nullable MemorySize managedMemory,
             Map<String, Double> extendedResources) {
-        this.name = checkNotNull(name);
-        this.cpuCores = checkNotNull(cpuCores);
-        this.taskHeapMemory = checkNotNull(taskHeapMemory);
-        this.taskOffHeapMemory = checkNotNull(taskOffHeapMemory);
-        this.managedMemory = checkNotNull(managedMemory);
-        this.externalResources.putAll(checkNotNull(extendedResources));
+        this.name = name;
+        this.cpuCores = cpuCores;
+        this.taskHeapMemory = taskHeapMemory;
+        this.taskOffHeapMemory = taskOffHeapMemory;
+        this.managedMemory = managedMemory;
+        this.externalResources = Collections.unmodifiableMap(extendedResources);
     }
 
     private SlotSharingGroup(String name) {
-        this.name = checkNotNull(name);
-        this.cpuCores = null;
-        this.taskHeapMemory = null;
-        this.taskOffHeapMemory = null;
-        this.managedMemory = null;
+        this(name, null, null, null, null, Collections.emptyMap());
     }
 
     public String getName() {
         return name;
     }
 
-    public Optional<MemorySize> getManagedMemory() {
-        return Optional.ofNullable(managedMemory);
+    @Nullable
+    public MemorySize getManagedMemory() {
+        return managedMemory;
     }
 
-    public Optional<MemorySize> getTaskHeapMemory() {
-        return Optional.ofNullable(taskHeapMemory);
+    @Nullable
+    public MemorySize getTaskHeapMemory() {
+        return taskHeapMemory;
     }
 
-    public Optional<MemorySize> getTaskOffHeapMemory() {
-        return Optional.ofNullable(taskOffHeapMemory);
+    @Nullable
+    public MemorySize getTaskOffHeapMemory() {
+        return taskOffHeapMemory;
     }
 
-    public Optional<Double> getCpuCores() {
-        return Optional.ofNullable(cpuCores)
-                .map(cpuResource -> cpuResource.getValue().doubleValue());
+    @Nullable
+    public Double getCpuCores() {
+        return cpuCores;
     }
 
     public Map<String, Double> getExternalResources() {
-        return Collections.unmodifiableMap(externalResources);
+        return externalResources;
     }
 
     public static Builder newBuilder(String name) {
@@ -137,15 +135,15 @@ public class SlotSharingGroup implements Serializable {
         return result;
     }
 
-    /** Builder for the {@link SlotSharingGroup}. */
+    /** Builder for {@link SlotSharingGroup}. */
+    @Experimental
     public static class Builder {
-
-        private String name;
-        private CPUResource cpuCores;
+        private final String name;
+        private Double cpuCores;
         private MemorySize taskHeapMemory;
         private MemorySize taskOffHeapMemory;
         private MemorySize managedMemory;
-        private Map<String, Double> externalResources = new HashMap<>();
+        private final Map<String, Double> externalResources = new HashMap<>();
 
         private Builder(String name) {
             this.name = name;
@@ -153,23 +151,27 @@ public class SlotSharingGroup implements Serializable {
 
         /** Set the CPU cores for this SlotSharingGroup. */
         public Builder setCpuCores(double cpuCores) {
-            checkArgument(cpuCores > 0, "The cpu cores should be positive.");
-            this.cpuCores = new CPUResource(cpuCores);
+            if (cpuCores <= 0) {
+                throw new IllegalArgumentException("The cpu cores should be positive.");
+            }
+            this.cpuCores = cpuCores;
             return this;
         }
 
         /** Set the task heap memory for this SlotSharingGroup. */
         public Builder setTaskHeapMemory(MemorySize taskHeapMemory) {
-            checkArgument(
-                    taskHeapMemory.compareTo(MemorySize.ZERO) > 0,
-                    "The task heap memory should be positive.");
+            if (taskHeapMemory.compareTo(MemorySize.ZERO) <= 0) {
+                throw new IllegalArgumentException("The task heap memory should be positive.");
+            }
             this.taskHeapMemory = taskHeapMemory;
             return this;
         }
 
         /** Set the task heap memory for this SlotSharingGroup in MB. */
         public Builder setTaskHeapMemoryMB(int taskHeapMemoryMB) {
-            checkArgument(taskHeapMemoryMB > 0, "The task heap memory should be positive.");
+            if (taskHeapMemoryMB <= 0) {
+                throw new IllegalArgumentException("The task heap memory should be positive.");
+            }
             this.taskHeapMemory = MemorySize.ofMebiBytes(taskHeapMemoryMB);
             return this;
         }
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java b/flink-core-api/src/main/java/org/apache/flink/configuration/MemorySize.java
similarity index 94%
rename from flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
rename to flink-core-api/src/main/java/org/apache/flink/configuration/MemorySize.java
index 4b718a749ff..d860c6e30fa 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
+++ b/flink-core-api/src/main/java/org/apache/flink/configuration/MemorySize.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Locale;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.IntStream;
 
@@ -34,8 +35,6 @@ import static org.apache.flink.configuration.MemorySize.MemoryUnit.KILO_BYTES;
 import static org.apache.flink.configuration.MemorySize.MemoryUnit.MEGA_BYTES;
 import static org.apache.flink.configuration.MemorySize.MemoryUnit.TERA_BYTES;
 import static org.apache.flink.configuration.MemorySize.MemoryUnit.hasUnit;
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkNotNull;
 
 /**
  * MemorySize is a representation of a number of bytes, viewable in different units.
@@ -74,7 +73,9 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
      * @param bytes The size, in bytes. Must be zero or larger.
      */
     public MemorySize(long bytes) {
-        checkArgument(bytes >= 0, "bytes must be >= 0");
+        if (bytes < 0) {
+            throw new IllegalArgumentException("bytes must be >= 0");
+        }
         this.bytes = bytes;
     }
 
@@ -204,7 +205,9 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
     }
 
     public MemorySize multiply(double multiplier) {
-        checkArgument(multiplier >= 0, "multiplier must be >= 0");
+        if (multiplier < 0) {
+            throw new IllegalArgumentException("multiplier must be >= 0");
+        }
 
         BigDecimal product =
                 BigDecimal.valueOf(this.bytes).multiply(BigDecimal.valueOf(multiplier));
@@ -215,7 +218,9 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
     }
 
     public MemorySize divide(long by) {
-        checkArgument(by >= 0, "divisor must be >= 0");
+        if (by < 0) {
+            throw new IllegalArgumentException("divisor must be != 0");
+        }
         return new MemorySize(bytes / by);
     }
 
@@ -260,10 +265,12 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
      * @throws IllegalArgumentException Thrown, if the expression cannot be parsed.
      */
     public static long parseBytes(String text) throws IllegalArgumentException {
-        checkNotNull(text, "text");
+        Objects.requireNonNull(text, "text cannot be null");
 
         final String trimmed = text.trim();
-        checkArgument(!trimmed.isEmpty(), "argument is an empty- or whitespace-only string");
+        if (trimmed.isEmpty()) {
+            throw new IllegalArgumentException("argument is an empty- or whitespace-only string");
+        }
 
         final int len = trimmed.length();
         int pos = 0;
@@ -382,10 +389,13 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
         }
 
         public static boolean hasUnit(String text) {
-            checkNotNull(text, "text");
+            Objects.requireNonNull(text, "text cannot be null");
 
             final String trimmed = text.trim();
-            checkArgument(!trimmed.isEmpty(), "argument is an empty- or whitespace-only string");
+            if (trimmed.isEmpty()) {
+                throw new IllegalArgumentException(
+                        "argument is an empty- or whitespace-only string");
+            }
 
             final int len = trimmed.length();
             int pos = 0;
diff --git a/flink-core-api/src/test/java/org/apache/flink/api/common/SlotSharingGroupTest.java b/flink-core-api/src/test/java/org/apache/flink/api/common/SlotSharingGroupTest.java
new file mode 100644
index 00000000000..f25625ffaa3
--- /dev/null
+++ b/flink-core-api/src/test/java/org/apache/flink/api/common/SlotSharingGroupTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common;
+
+import org.apache.flink.configuration.MemorySize;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for {@link SlotSharingGroup}. */
+class SlotSharingGroupTest {
+    @Test
+    void testBuildSlotSharingGroupWithSpecificResource() {
+        final String name = "ssg";
+        final MemorySize heap = MemorySize.ofMebiBytes(100);
+        final MemorySize offHeap = MemorySize.ofMebiBytes(200);
+        final MemorySize managed = MemorySize.ofMebiBytes(300);
+        final SlotSharingGroup slotSharingGroup =
+                SlotSharingGroup.newBuilder(name)
+                        .setCpuCores(1)
+                        .setTaskHeapMemory(heap)
+                        .setTaskOffHeapMemory(offHeap)
+                        .setManagedMemory(managed)
+                        .setExternalResource("gpu", 1)
+                        .build();
+
+        assertThat(slotSharingGroup.getName()).isEqualTo(name);
+        assertThat(slotSharingGroup.getCpuCores()).isEqualTo(1.0);
+        assertThat(slotSharingGroup.getTaskHeapMemory()).isEqualTo(heap);
+        assertThat(slotSharingGroup.getTaskOffHeapMemory()).isEqualTo(offHeap);
+        assertThat(slotSharingGroup.getManagedMemory()).isEqualTo(managed);
+        assertThat(slotSharingGroup.getExternalResources())
+                .isEqualTo(Collections.singletonMap("gpu", 1.0));
+    }
+
+    @Test
+    void testBuildSlotSharingGroupWithUnknownResource() {
+        final String name = "ssg";
+        final SlotSharingGroup slotSharingGroup = SlotSharingGroup.newBuilder(name).build();
+
+        assertThat(slotSharingGroup.getName()).isEqualTo(name);
+        assertThat(slotSharingGroup.getCpuCores()).isNull();
+        assertThat(slotSharingGroup.getTaskHeapMemory()).isNull();
+        assertThat(slotSharingGroup.getManagedMemory()).isNull();
+        assertThat(slotSharingGroup.getTaskOffHeapMemory()).isNull();
+        assertThat(slotSharingGroup.getExternalResources()).isEmpty();
+    }
+
+    @Test
+    void testBuildSlotSharingGroupWithIllegalConfig() {
+        assertThatThrownBy(
+                        () ->
+                                SlotSharingGroup.newBuilder("ssg")
+                                        .setCpuCores(1)
+                                        .setTaskHeapMemory(MemorySize.ZERO)
+                                        .setTaskOffHeapMemoryMB(10)
+                                        .build())
+                .isInstanceOf(IllegalArgumentException.class);
+    }
+
+    @Test
+    void testBuildSlotSharingGroupWithoutAllRequiredConfig() {
+        assertThatThrownBy(
+                        () ->
+                                SlotSharingGroup.newBuilder("ssg")
+                                        .setCpuCores(1)
+                                        .setTaskOffHeapMemoryMB(10)
+                                        .build())
+                .isInstanceOf(IllegalArgumentException.class);
+    }
+}
diff --git a/flink-core/src/test/java/org/apache/flink/configuration/MemorySizePrettyPrintingTest.java b/flink-core-api/src/test/java/org/apache/flink/configuration/MemorySizePrettyPrintingTest.java
similarity index 100%
rename from flink-core/src/test/java/org/apache/flink/configuration/MemorySizePrettyPrintingTest.java
rename to flink-core-api/src/test/java/org/apache/flink/configuration/MemorySizePrettyPrintingTest.java
diff --git a/flink-core/src/test/java/org/apache/flink/configuration/MemorySizeTest.java b/flink-core-api/src/test/java/org/apache/flink/configuration/MemorySizeTest.java
similarity index 100%
rename from flink-core/src/test/java/org/apache/flink/configuration/MemorySizeTest.java
rename to flink-core-api/src/test/java/org/apache/flink/configuration/MemorySizeTest.java
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java
index 7a24dcfde57..26fd6303b2d 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java
@@ -137,6 +137,37 @@ public class SlotSharingGroup implements Serializable {
         return result;
     }
 
+    /**
+     * Convert a {@link org.apache.flink.api.common.SlotSharingGroup} to {@link SlotSharingGroup}.
+     */
+    public static SlotSharingGroup from(org.apache.flink.api.common.SlotSharingGroup group) {
+        if (group.getCpuCores() != null && group.getTaskHeapMemory() != null) {
+            MemorySize taskOffHeapMemory =
+                    group.getTaskOffHeapMemory() == null
+                            ? MemorySize.ZERO
+                            : group.getTaskOffHeapMemory();
+            MemorySize managedMemory =
+                    group.getManagedMemory() == null ? MemorySize.ZERO : group.getManagedMemory();
+            return new SlotSharingGroup(
+                    group.getName(),
+                    new CPUResource(group.getCpuCores()),
+                    group.getTaskHeapMemory(),
+                    taskOffHeapMemory,
+                    managedMemory,
+                    group.getExternalResources());
+        } else if (group.getCpuCores() != null
+                || group.getTaskHeapMemory() != null
+                || group.getTaskOffHeapMemory() != null
+                || group.getManagedMemory() != null
+                || !group.getExternalResources().isEmpty()) {
+            throw new IllegalArgumentException(
+                    "The cpu cores and task heap memory are required when specifying the resource of a slot sharing group. "
+                            + "You need to explicitly configure them with positive value.");
+        } else {
+            return new SlotSharingGroup(group.getName());
+        }
+    }
+
     /** Builder for the {@link SlotSharingGroup}. */
     public static class Builder {
 
diff --git a/flink-table/flink-sql-jdbc-driver-bundle/pom.xml b/flink-table/flink-sql-jdbc-driver-bundle/pom.xml
index da6bf5d4ab3..d296738d9e6 100644
--- a/flink-table/flink-sql-jdbc-driver-bundle/pom.xml
+++ b/flink-table/flink-sql-jdbc-driver-bundle/pom.xml
@@ -111,6 +111,7 @@
 									<include>org.apache.flink:flink-table-common</include>
 									<include>org.apache.flink:flink-annotations</include>
 									<include>org.apache.flink:flink-core</include>
+									<include>org.apache.flink:flink-core-api</include>
 									<include>org.apache.flink:flink-runtime</include>
 									<include>org.apache.flink:flink-clients</include>
 									<include>org.apache.flink:flink-table-api-java</include>