You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gu...@apache.org on 2024/05/22 04:04:37 UTC
(flink) 02/11: [FLINK-34549][API] Introduce SlotSharingGroup for v2 and move MemorySize to flink-core-api
This is an automated email from the ASF dual-hosted git repository.
guoweijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 1d73510bc2e30d5352a8cbbafd7939197a32cedb
Author: Weijie Guo <re...@163.com>
AuthorDate: Tue Apr 16 14:42:48 2024 +0800
[FLINK-34549][API] Introduce SlotSharingGroup for v2 and move MemorySize to flink-core-api
---
.../5b9eed8a-5fb6-4373-98ac-3be2a71941b8 | 1 +
flink-core-api/pom.xml | 7 ++
.../apache/flink/api/common}/SlotSharingGroup.java | 102 +++++++++++----------
.../org/apache/flink/configuration/MemorySize.java | 28 ++++--
.../flink/api/common/SlotSharingGroupTest.java | 91 ++++++++++++++++++
.../MemorySizePrettyPrintingTest.java | 0
.../apache/flink/configuration/MemorySizeTest.java | 0
.../api/common/operators/SlotSharingGroup.java | 31 +++++++
flink-table/flink-sql-jdbc-driver-bundle/pom.xml | 1 +
9 files changed, 202 insertions(+), 59 deletions(-)
diff --git a/flink-architecture-tests/flink-architecture-tests-production/archunit-violations/5b9eed8a-5fb6-4373-98ac-3be2a71941b8 b/flink-architecture-tests/flink-architecture-tests-production/archunit-violations/5b9eed8a-5fb6-4373-98ac-3be2a71941b8
index 2c83601f889..a40e8acda25 100644
--- a/flink-architecture-tests/flink-architecture-tests-production/archunit-violations/5b9eed8a-5fb6-4373-98ac-3be2a71941b8
+++ b/flink-architecture-tests/flink-architecture-tests-production/archunit-violations/5b9eed8a-5fb6-4373-98ac-3be2a71941b8
@@ -3,6 +3,7 @@ org.apache.flink.api.common.eventtime.WatermarkGeneratorSupplier.createWatermark
org.apache.flink.api.common.operators.Operator.getMinResources(): Returned leaf type org.apache.flink.api.common.operators.ResourceSpec does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
org.apache.flink.api.common.operators.Operator.getPreferredResources(): Returned leaf type org.apache.flink.api.common.operators.ResourceSpec does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
org.apache.flink.api.common.operators.Operator.setResources(org.apache.flink.api.common.operators.ResourceSpec, org.apache.flink.api.common.operators.ResourceSpec): Argument leaf type org.apache.flink.api.common.operators.ResourceSpec does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
+org.apache.flink.api.common.operators.SlotSharingGroup.from(org.apache.flink.api.common.SlotSharingGroup): Argument leaf type org.apache.flink.api.common.SlotSharingGroup does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
org.apache.flink.api.common.operators.SlotSharingGroup.newBuilder(java.lang.String): Returned leaf type org.apache.flink.api.common.operators.SlotSharingGroup$Builder does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
org.apache.flink.api.common.state.AggregatingStateDescriptor.getType(): Returned leaf type org.apache.flink.api.common.state.StateDescriptor$Type does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
org.apache.flink.api.common.state.ListStateDescriptor.getType(): Returned leaf type org.apache.flink.api.common.state.StateDescriptor$Type does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
diff --git a/flink-core-api/pom.xml b/flink-core-api/pom.xml
index 8b95bc57e8e..a66ca8b386a 100644
--- a/flink-core-api/pom.xml
+++ b/flink-core-api/pom.xml
@@ -39,5 +39,12 @@ under the License.
<artifactId>flink-annotations</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <!-- ================== test dependencies ================== -->
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils-junit</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java b/flink-core-api/src/main/java/org/apache/flink/api/common/SlotSharingGroup.java
similarity index 72%
copy from flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java
copy to flink-core-api/src/main/java/org/apache/flink/api/common/SlotSharingGroup.java
index 7a24dcfde57..fc8268cf9e7 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java
+++ b/flink-core-api/src/main/java/org/apache/flink/api/common/SlotSharingGroup.java
@@ -16,34 +16,33 @@
* limitations under the License.
*/
-package org.apache.flink.api.common.operators;
+package org.apache.flink.api.common;
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.resources.CPUResource;
+import org.apache.flink.annotation.Experimental;
import org.apache.flink.configuration.MemorySize;
import javax.annotation.Nullable;
-import java.io.Serializable;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/** Describe the name and the different resource components of a slot sharing group. */
-@PublicEvolving
-public class SlotSharingGroup implements Serializable {
- private static final long serialVersionUID = 1L;
-
+/**
+ * Describe the name and the different resource components of a slot sharing group.
+ *
+ * <p>Two SlotSharingGroup classes currently exist in flink-core and flink-core-api, because the one
+ * in flink-core exposes components marked as internal, which we want to avoid in DataStream-V2 API.
+ * Once the V1 API is removed, we will remove the legacy SlotSharingGroup.
+ */
+@Experimental
+public class SlotSharingGroup {
private final String name;
/** How many cpu cores are needed. Can be null only if it is unknown. */
@Nullable // can be null only for UNKNOWN
- private final CPUResource cpuCores;
+ private final Double cpuCores;
/** How much task heap memory is needed. */
@Nullable // can be null only for UNKNOWN
@@ -58,54 +57,53 @@ public class SlotSharingGroup implements Serializable {
private final MemorySize managedMemory;
/** A extensible field for user specified resources from {@link SlotSharingGroup}. */
- private final Map<String, Double> externalResources = new HashMap<>();
+ private final Map<String, Double> externalResources;
private SlotSharingGroup(
String name,
- CPUResource cpuCores,
- MemorySize taskHeapMemory,
- MemorySize taskOffHeapMemory,
- MemorySize managedMemory,
+ @Nullable Double cpuCores,
+ @Nullable MemorySize taskHeapMemory,
+ @Nullable MemorySize taskOffHeapMemory,
+ @Nullable MemorySize managedMemory,
Map<String, Double> extendedResources) {
- this.name = checkNotNull(name);
- this.cpuCores = checkNotNull(cpuCores);
- this.taskHeapMemory = checkNotNull(taskHeapMemory);
- this.taskOffHeapMemory = checkNotNull(taskOffHeapMemory);
- this.managedMemory = checkNotNull(managedMemory);
- this.externalResources.putAll(checkNotNull(extendedResources));
+ this.name = name;
+ this.cpuCores = cpuCores;
+ this.taskHeapMemory = taskHeapMemory;
+ this.taskOffHeapMemory = taskOffHeapMemory;
+ this.managedMemory = managedMemory;
+ this.externalResources = Collections.unmodifiableMap(extendedResources);
}
private SlotSharingGroup(String name) {
- this.name = checkNotNull(name);
- this.cpuCores = null;
- this.taskHeapMemory = null;
- this.taskOffHeapMemory = null;
- this.managedMemory = null;
+ this(name, null, null, null, null, Collections.emptyMap());
}
public String getName() {
return name;
}
- public Optional<MemorySize> getManagedMemory() {
- return Optional.ofNullable(managedMemory);
+ @Nullable
+ public MemorySize getManagedMemory() {
+ return managedMemory;
}
- public Optional<MemorySize> getTaskHeapMemory() {
- return Optional.ofNullable(taskHeapMemory);
+ @Nullable
+ public MemorySize getTaskHeapMemory() {
+ return taskHeapMemory;
}
- public Optional<MemorySize> getTaskOffHeapMemory() {
- return Optional.ofNullable(taskOffHeapMemory);
+ @Nullable
+ public MemorySize getTaskOffHeapMemory() {
+ return taskOffHeapMemory;
}
- public Optional<Double> getCpuCores() {
- return Optional.ofNullable(cpuCores)
- .map(cpuResource -> cpuResource.getValue().doubleValue());
+ @Nullable
+ public Double getCpuCores() {
+ return cpuCores;
}
public Map<String, Double> getExternalResources() {
- return Collections.unmodifiableMap(externalResources);
+ return externalResources;
}
public static Builder newBuilder(String name) {
@@ -137,15 +135,15 @@ public class SlotSharingGroup implements Serializable {
return result;
}
- /** Builder for the {@link SlotSharingGroup}. */
+ /** Builder for {@link SlotSharingGroup}. */
+ @Experimental
public static class Builder {
-
- private String name;
- private CPUResource cpuCores;
+ private final String name;
+ private Double cpuCores;
private MemorySize taskHeapMemory;
private MemorySize taskOffHeapMemory;
private MemorySize managedMemory;
- private Map<String, Double> externalResources = new HashMap<>();
+ private final Map<String, Double> externalResources = new HashMap<>();
private Builder(String name) {
this.name = name;
@@ -153,23 +151,27 @@ public class SlotSharingGroup implements Serializable {
/** Set the CPU cores for this SlotSharingGroup. */
public Builder setCpuCores(double cpuCores) {
- checkArgument(cpuCores > 0, "The cpu cores should be positive.");
- this.cpuCores = new CPUResource(cpuCores);
+ if (cpuCores <= 0) {
+ throw new IllegalArgumentException("The cpu cores should be positive.");
+ }
+ this.cpuCores = cpuCores;
return this;
}
/** Set the task heap memory for this SlotSharingGroup. */
public Builder setTaskHeapMemory(MemorySize taskHeapMemory) {
- checkArgument(
- taskHeapMemory.compareTo(MemorySize.ZERO) > 0,
- "The task heap memory should be positive.");
+ if (taskHeapMemory.compareTo(MemorySize.ZERO) <= 0) {
+ throw new IllegalArgumentException("The task heap memory should be positive.");
+ }
this.taskHeapMemory = taskHeapMemory;
return this;
}
/** Set the task heap memory for this SlotSharingGroup in MB. */
public Builder setTaskHeapMemoryMB(int taskHeapMemoryMB) {
- checkArgument(taskHeapMemoryMB > 0, "The task heap memory should be positive.");
+ if (taskHeapMemoryMB <= 0) {
+ throw new IllegalArgumentException("The task heap memory should be positive.");
+ }
this.taskHeapMemory = MemorySize.ofMebiBytes(taskHeapMemoryMB);
return this;
}
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java b/flink-core-api/src/main/java/org/apache/flink/configuration/MemorySize.java
similarity index 94%
rename from flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
rename to flink-core-api/src/main/java/org/apache/flink/configuration/MemorySize.java
index 4b718a749ff..d860c6e30fa 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
+++ b/flink-core-api/src/main/java/org/apache/flink/configuration/MemorySize.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
+import java.util.Objects;
import java.util.Optional;
import java.util.stream.IntStream;
@@ -34,8 +35,6 @@ import static org.apache.flink.configuration.MemorySize.MemoryUnit.KILO_BYTES;
import static org.apache.flink.configuration.MemorySize.MemoryUnit.MEGA_BYTES;
import static org.apache.flink.configuration.MemorySize.MemoryUnit.TERA_BYTES;
import static org.apache.flink.configuration.MemorySize.MemoryUnit.hasUnit;
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* MemorySize is a representation of a number of bytes, viewable in different units.
@@ -74,7 +73,9 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
* @param bytes The size, in bytes. Must be zero or larger.
*/
public MemorySize(long bytes) {
- checkArgument(bytes >= 0, "bytes must be >= 0");
+ if (bytes < 0) {
+ throw new IllegalArgumentException("bytes must be >= 0");
+ }
this.bytes = bytes;
}
@@ -204,7 +205,9 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
}
public MemorySize multiply(double multiplier) {
- checkArgument(multiplier >= 0, "multiplier must be >= 0");
+ if (multiplier < 0) {
+ throw new IllegalArgumentException("multiplier must be >= 0");
+ }
BigDecimal product =
BigDecimal.valueOf(this.bytes).multiply(BigDecimal.valueOf(multiplier));
@@ -215,7 +218,9 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
}
public MemorySize divide(long by) {
- checkArgument(by >= 0, "divisor must be >= 0");
+ if (by < 0) {
+ throw new IllegalArgumentException("divisor must be != 0");
+ }
return new MemorySize(bytes / by);
}
@@ -260,10 +265,12 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
* @throws IllegalArgumentException Thrown, if the expression cannot be parsed.
*/
public static long parseBytes(String text) throws IllegalArgumentException {
- checkNotNull(text, "text");
+ Objects.requireNonNull(text, "text cannot be null");
final String trimmed = text.trim();
- checkArgument(!trimmed.isEmpty(), "argument is an empty- or whitespace-only string");
+ if (trimmed.isEmpty()) {
+ throw new IllegalArgumentException("argument is an empty- or whitespace-only string");
+ }
final int len = trimmed.length();
int pos = 0;
@@ -382,10 +389,13 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
}
public static boolean hasUnit(String text) {
- checkNotNull(text, "text");
+ Objects.requireNonNull(text, "text cannot be null");
final String trimmed = text.trim();
- checkArgument(!trimmed.isEmpty(), "argument is an empty- or whitespace-only string");
+ if (trimmed.isEmpty()) {
+ throw new IllegalArgumentException(
+ "argument is an empty- or whitespace-only string");
+ }
final int len = trimmed.length();
int pos = 0;
diff --git a/flink-core-api/src/test/java/org/apache/flink/api/common/SlotSharingGroupTest.java b/flink-core-api/src/test/java/org/apache/flink/api/common/SlotSharingGroupTest.java
new file mode 100644
index 00000000000..f25625ffaa3
--- /dev/null
+++ b/flink-core-api/src/test/java/org/apache/flink/api/common/SlotSharingGroupTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common;
+
+import org.apache.flink.configuration.MemorySize;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for {@link SlotSharingGroup}. */
+class SlotSharingGroupTest {
+ @Test
+ void testBuildSlotSharingGroupWithSpecificResource() {
+ final String name = "ssg";
+ final MemorySize heap = MemorySize.ofMebiBytes(100);
+ final MemorySize offHeap = MemorySize.ofMebiBytes(200);
+ final MemorySize managed = MemorySize.ofMebiBytes(300);
+ final SlotSharingGroup slotSharingGroup =
+ SlotSharingGroup.newBuilder(name)
+ .setCpuCores(1)
+ .setTaskHeapMemory(heap)
+ .setTaskOffHeapMemory(offHeap)
+ .setManagedMemory(managed)
+ .setExternalResource("gpu", 1)
+ .build();
+
+ assertThat(slotSharingGroup.getName()).isEqualTo(name);
+ assertThat(slotSharingGroup.getCpuCores()).isEqualTo(1.0);
+ assertThat(slotSharingGroup.getTaskHeapMemory()).isEqualTo(heap);
+ assertThat(slotSharingGroup.getTaskOffHeapMemory()).isEqualTo(offHeap);
+ assertThat(slotSharingGroup.getManagedMemory()).isEqualTo(managed);
+ assertThat(slotSharingGroup.getExternalResources())
+ .isEqualTo(Collections.singletonMap("gpu", 1.0));
+ }
+
+ @Test
+ void testBuildSlotSharingGroupWithUnknownResource() {
+ final String name = "ssg";
+ final SlotSharingGroup slotSharingGroup = SlotSharingGroup.newBuilder(name).build();
+
+ assertThat(slotSharingGroup.getName()).isEqualTo(name);
+ assertThat(slotSharingGroup.getCpuCores()).isNull();
+ assertThat(slotSharingGroup.getTaskHeapMemory()).isNull();
+ assertThat(slotSharingGroup.getManagedMemory()).isNull();
+ assertThat(slotSharingGroup.getTaskOffHeapMemory()).isNull();
+ assertThat(slotSharingGroup.getExternalResources()).isEmpty();
+ }
+
+ @Test
+ void testBuildSlotSharingGroupWithIllegalConfig() {
+ assertThatThrownBy(
+ () ->
+ SlotSharingGroup.newBuilder("ssg")
+ .setCpuCores(1)
+ .setTaskHeapMemory(MemorySize.ZERO)
+ .setTaskOffHeapMemoryMB(10)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void testBuildSlotSharingGroupWithoutAllRequiredConfig() {
+ assertThatThrownBy(
+ () ->
+ SlotSharingGroup.newBuilder("ssg")
+ .setCpuCores(1)
+ .setTaskOffHeapMemoryMB(10)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+}
diff --git a/flink-core/src/test/java/org/apache/flink/configuration/MemorySizePrettyPrintingTest.java b/flink-core-api/src/test/java/org/apache/flink/configuration/MemorySizePrettyPrintingTest.java
similarity index 100%
rename from flink-core/src/test/java/org/apache/flink/configuration/MemorySizePrettyPrintingTest.java
rename to flink-core-api/src/test/java/org/apache/flink/configuration/MemorySizePrettyPrintingTest.java
diff --git a/flink-core/src/test/java/org/apache/flink/configuration/MemorySizeTest.java b/flink-core-api/src/test/java/org/apache/flink/configuration/MemorySizeTest.java
similarity index 100%
rename from flink-core/src/test/java/org/apache/flink/configuration/MemorySizeTest.java
rename to flink-core-api/src/test/java/org/apache/flink/configuration/MemorySizeTest.java
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java
index 7a24dcfde57..26fd6303b2d 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/SlotSharingGroup.java
@@ -137,6 +137,37 @@ public class SlotSharingGroup implements Serializable {
return result;
}
+ /**
+ * Convert a {@link org.apache.flink.api.common.SlotSharingGroup} to {@link SlotSharingGroup}.
+ */
+ public static SlotSharingGroup from(org.apache.flink.api.common.SlotSharingGroup group) {
+ if (group.getCpuCores() != null && group.getTaskHeapMemory() != null) {
+ MemorySize taskOffHeapMemory =
+ group.getTaskOffHeapMemory() == null
+ ? MemorySize.ZERO
+ : group.getTaskOffHeapMemory();
+ MemorySize managedMemory =
+ group.getManagedMemory() == null ? MemorySize.ZERO : group.getManagedMemory();
+ return new SlotSharingGroup(
+ group.getName(),
+ new CPUResource(group.getCpuCores()),
+ group.getTaskHeapMemory(),
+ taskOffHeapMemory,
+ managedMemory,
+ group.getExternalResources());
+ } else if (group.getCpuCores() != null
+ || group.getTaskHeapMemory() != null
+ || group.getTaskOffHeapMemory() != null
+ || group.getManagedMemory() != null
+ || !group.getExternalResources().isEmpty()) {
+ throw new IllegalArgumentException(
+ "The cpu cores and task heap memory are required when specifying the resource of a slot sharing group. "
+ + "You need to explicitly configure them with positive value.");
+ } else {
+ return new SlotSharingGroup(group.getName());
+ }
+ }
+
/** Builder for the {@link SlotSharingGroup}. */
public static class Builder {
diff --git a/flink-table/flink-sql-jdbc-driver-bundle/pom.xml b/flink-table/flink-sql-jdbc-driver-bundle/pom.xml
index da6bf5d4ab3..d296738d9e6 100644
--- a/flink-table/flink-sql-jdbc-driver-bundle/pom.xml
+++ b/flink-table/flink-sql-jdbc-driver-bundle/pom.xml
@@ -111,6 +111,7 @@
<include>org.apache.flink:flink-table-common</include>
<include>org.apache.flink:flink-annotations</include>
<include>org.apache.flink:flink-core</include>
+ <include>org.apache.flink:flink-core-api</include>
<include>org.apache.flink:flink-runtime</include>
<include>org.apache.flink:flink-clients</include>
<include>org.apache.flink:flink-table-api-java</include>