You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/10/17 06:12:31 UTC
(pulsar) 12/14: [improve][fn] Provide a way to set Pulsar client memory limit in Pulsar Function (#23959)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 5f320772644f30ad7cfc1caa3c58ca058a9581b7
Author: Lee hong <40...@users.noreply.github.com>
AuthorDate: Sat Oct 17 12:43:33 2026 +0800
[improve][fn] Provide a way to set Pulsar client memory limit in Pulsar Function (#23959)
Co-authored-by: Lari Hotari <lh...@users.noreply.github.com>
Co-authored-by: Matteo Merli <mm...@apache.org>
(cherry picked from commit 2fbd8153e09508c23e18b2e9514a9e44c67e794c)
---
conf/functions_worker.yml | 9 ++++
.../pulsar/common/functions/FunctionConfig.java | 3 ++
.../pulsar/common/functions/MemoryLimit.java | 31 ++++++-----
.../apache/pulsar/admin/cli/CmdFunctionsTest.java | 22 ++++++++
.../org/apache/pulsar/admin/cli/CmdFunctions.java | 15 ++++++
.../pulsar/common/util/DirectMemoryUtils.java | 3 ++
.../pulsar/functions/instance/InstanceConfig.java | 3 +-
.../org/apache/pulsar/functions/LocalRunner.java | 8 ++-
.../proto/src/main/proto/Function.proto | 9 ++++
.../functions/runtime/JavaInstanceStarter.java | 19 ++++++-
.../pulsar/functions/runtime/RuntimeUtils.java | 11 ++++
.../runtime/thread/ThreadRuntimeFactory.java | 35 ++++++++-----
.../pulsar/functions/worker/WorkerConfig.java | 10 ++++
.../pulsar/functions/runtime/RuntimeUtilsTest.java | 42 +++++++++++++++
.../runtime/thread/ThreadRuntimeFactoryTest.java | 43 +++++++++++----
.../functions/utils/FunctionConfigUtils.java | 45 ++++++++++++++++
.../functions/utils/FunctionConfigUtilsTest.java | 61 ++++++++++++++++++++++
.../pulsar/functions/worker/FunctionActioner.java | 4 ++
.../functions/worker/FunctionActionerTest.java | 27 ++++++++++
19 files changed, 358 insertions(+), 42 deletions(-)
diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml
index 426ae802913c..1a13e361b13d 100644
--- a/conf/functions_worker.yml
+++ b/conf/functions_worker.yml
@@ -169,6 +169,15 @@ functionRuntimeFactoryConfigs:
# This arguments will be added to the command line execution of 'java'
#additionalJavaRuntimeArguments: ['-XX:+ExitOnOutOfMemoryError']
+#### Pulsar client memory limit of Java function instances ####
+# Default for functions that don't set their own limit. If both values are set, the smaller limit is used.
+# The thread runtime uses it when functionRuntimeFactoryConfigs.pulsarClientMemoryLimit is not set.
+#pulsarClientMemoryLimit:
+# # the max memory in bytes the pulsar client can use
+# limitInBytes:
+# # the max memory the pulsar client can use as a percentage of max direct memory set for JVM
+# percentOfMaxDirectMemory:
+
#### Thread Runtime ####
# Pulsar function instances are run as threads
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java
index 9d6ce1fcc0d3..ce791caea02a 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java
@@ -147,6 +147,9 @@ public class FunctionConfig {
private Boolean skipToLatest;
// Pulsar client API for the function's own topics; null picks it from the topic domains
private ClientApi clientApi;
+ // Memory limit of the Pulsar client used by each Java function instance; null uses the worker default.
+ // Ignored by the worker's thread runtime, where all instances share one client configured by the worker.
+ private MemoryLimit pulsarClientMemoryLimit;
@Builder.Default
private SubscriptionInitialPosition subscriptionPosition = SubscriptionInitialPosition.Latest;
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/DirectMemoryUtils.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/MemoryLimit.java
similarity index 57%
copy from pulsar-common/src/main/java/org/apache/pulsar/common/util/DirectMemoryUtils.java
copy to pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/MemoryLimit.java
index 0e5a450aa6bf..01cf706acda1 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/DirectMemoryUtils.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/MemoryLimit.java
@@ -16,22 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pulsar.common.util;
+package org.apache.pulsar.common.functions;
-// CHECKSTYLE.OFF: IllegalImport
-import io.netty.util.internal.PlatformDependent;
-// CHECKSTYLE.ON: IllegalImport
-
-public class DirectMemoryUtils {
-
- /**
- * PlatformDependent.maxDirectMemory can be inaccurate if java property `io.netty.maxDirectMemory` are setted.
- * Cache the result in this field.
- */
- private static final long JVM_MAX_DIRECT_MEMORY = PlatformDependent.estimateMaxDirectMemory();
-
- public static long jvmMaxDirectMemory() {
- return JVM_MAX_DIRECT_MEMORY;
- }
+import lombok.Data;
+import lombok.experimental.Accessors;
+/**
+ * Memory limit of the Pulsar client used by a function instance.
+ * If both {@code limitInBytes} and {@code percentOfMaxDirectMemory} are set,
+ * the smaller of the two limits is used.
+ */
+@Data
+@Accessors(chain = true)
+public class MemoryLimit {
+ // The max memory in bytes the Pulsar client can use
+ private Long limitInBytes;
+ // The max memory the Pulsar client can use, as a percentage (0, 100] of the JVM max direct memory
+ private Double percentOfMaxDirectMemory;
}
diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java
index 9950bf257b7f..5488f0bd1654 100644
--- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java
+++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java
@@ -45,6 +45,7 @@ import org.apache.pulsar.admin.cli.CmdFunctions.UpdateFunction;
import org.apache.pulsar.client.admin.Functions;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.common.functions.FunctionConfig;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.functions.UpdateOptionsImpl;
import org.apache.pulsar.functions.api.Context;
import org.apache.pulsar.functions.api.Function;
@@ -519,6 +520,27 @@ public class CmdFunctionsTest {
verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString());
}
+ @Test
+ public void testCreateFunctionWithPulsarClientMemoryLimit() throws Exception {
+ cmd.run(new String[] {
+ "create",
+ "--name", FN_NAME,
+ "--inputs", "persistent://sample/ns1/in",
+ "--output", "persistent://sample/ns1/out",
+ "--client-memory-limit", "64M",
+ "--client-memory-limit-percent", "25",
+ "--jar", JAR_NAME,
+ "--tenant", "sample",
+ "--namespace", "ns1",
+ "--className", DummyFunction.class.getName(),
+ });
+
+ CreateFunction creater = cmd.getCreater();
+ assertEquals(new MemoryLimit().setLimitInBytes(64L * 1024 * 1024).setPercentOfMaxDirectMemory(25.0),
+ creater.getFunctionConfig().getPulsarClientMemoryLimit());
+ verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString());
+ }
+
@Test
public void testCreateUsingFullyQualifiedFunctionName() throws Exception {
String tenant = "sample";
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
index 330c1e0f1c5f..8b57f1ae1bac 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
@@ -42,6 +42,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.WordUtils;
import org.apache.pulsar.admin.cli.utils.CmdUtils;
import org.apache.pulsar.cli.ClientApi;
+import org.apache.pulsar.cli.converters.picocli.ByteUnitToLongConverter;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.api.PulsarClientException;
@@ -49,6 +50,7 @@ import org.apache.pulsar.client.api.SubscriptionInitialPosition;
import org.apache.pulsar.common.functions.ConsumerConfig;
import org.apache.pulsar.common.functions.FunctionConfig;
import org.apache.pulsar.common.functions.FunctionState;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.functions.ProducerConfig;
import org.apache.pulsar.common.functions.Resources;
import org.apache.pulsar.common.functions.UpdateOptionsImpl;
@@ -325,6 +327,13 @@ public class CmdFunctions extends CmdBase {
protected Boolean skipToLatest;
@Option(names = ClientApi.OPTION_NAME, description = CmdFunctions.CLIENT_API_DESCRIPTION)
protected FunctionConfig.ClientApi clientApi;
+ @Option(names = "--client-memory-limit", description = "The max memory the Pulsar client of each function "
+ + "instance can use (eg: 32M, 64M) #Java(Process & K8s)", converter = ByteUnitToLongConverter.class)
+ protected Long clientMemoryLimit;
+ @Option(names = "--client-memory-limit-percent", description = "The max memory the Pulsar client of each "
+ + "function instance can use, as a percentage of the instance JVM max direct memory. If "
+ + "--client-memory-limit is also set, the smaller of the two limits is used #Java(Process & K8s)")
+ protected Double clientMemoryLimitPercent;
@Option(names = "--parallelism", description = "The parallelism factor of a Pulsar Function "
+ "(i.e. the number of function instances to run) #Java")
protected Integer parallelism;
@@ -571,6 +580,12 @@ public class CmdFunctions extends CmdBase {
functionConfig.setClientApi(clientApi);
}
+ if (null != clientMemoryLimit || null != clientMemoryLimitPercent) {
+ functionConfig.setPulsarClientMemoryLimit(new MemoryLimit()
+ .setLimitInBytes(clientMemoryLimit)
+ .setPercentOfMaxDirectMemory(clientMemoryLimitPercent));
+ }
+
if (null != userConfigString) {
Type type = new TypeToken<Map<String, Object>>() {}.getType();
Map<String, Object> userConfigMap = new Gson().fromJson(userConfigString, type);
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/DirectMemoryUtils.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/DirectMemoryUtils.java
index 0e5a450aa6bf..abc1466922ea 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/DirectMemoryUtils.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/DirectMemoryUtils.java
@@ -34,4 +34,7 @@ public class DirectMemoryUtils {
return JVM_MAX_DIRECT_MEMORY;
}
+ public static long jvmMaxDirectMemoryPercentage(double percent) {
+ return (long) (DirectMemoryUtils.jvmMaxDirectMemory() * (percent / 100));
+ }
}
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java
index f4c5ca380860..204ebadb6418 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java
@@ -22,6 +22,7 @@ import java.util.Collections;
import java.util.List;
import lombok.Data;
import lombok.Getter;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.functions.proto.FunctionAuthenticationSpec;
import org.apache.pulsar.functions.proto.FunctionDetails;
@@ -49,7 +50,7 @@ public class InstanceConfig {
private int metricsPort;
private List<String> additionalJavaRuntimeArguments = Collections.emptyList();
private boolean ignoreUnknownConfigFields;
-
+ private MemoryLimit pulsarClientMemoryLimit;
/**
* Get the string representation of {@link #getInstanceId()}.
*
diff --git a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java
index f7f524aa495a..ad169a160970 100644
--- a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java
+++ b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java
@@ -49,6 +49,7 @@ import lombok.CustomLog;
import lombok.Value;
import org.apache.pulsar.common.functions.FunctionConfig;
import org.apache.pulsar.common.functions.FunctionDefinition;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.functions.Utils;
import org.apache.pulsar.common.io.ConnectorDefinition;
import org.apache.pulsar.common.io.SinkConfig;
@@ -582,6 +583,7 @@ public class LocalRunner implements AutoCloseable {
instanceConfig
.setExposePulsarAdminClientEnabled(functionConfig.getExposePulsarAdminClientEnabled());
}
+ instanceConfig.setPulsarClientMemoryLimit(functionConfig.getPulsarClientMemoryLimit());
}
RuntimeSpawner runtimeSpawner = new RuntimeSpawner(
@@ -662,6 +664,9 @@ public class LocalRunner implements AutoCloseable {
if (userCodeClassLoader != null && userCodeClassLoader.getClassLoader() != null) {
Thread.currentThread().setContextClassLoader(userCodeClassLoader.getClassLoader());
}
+ Optional<MemoryLimit> pulsarClientMemoryLimit = functionConfig != null
+ ? Optional.ofNullable(functionConfig.getPulsarClientMemoryLimit())
+ : Optional.empty();
runtimeFactory = new ThreadRuntimeFactory("LocalRunnerThreadGroup",
serviceUrl,
stateStorageImplClass,
@@ -670,7 +675,7 @@ public class LocalRunner implements AutoCloseable {
secretsProvider,
collectorRegistry, narExtractionDirectory,
null,
- exposePulsarAdminClientEnabled, webServiceUrl);
+ exposePulsarAdminClientEnabled, webServiceUrl, pulsarClientMemoryLimit);
} finally {
Thread.currentThread().setContextClassLoader(originalClassLoader);
}
@@ -693,6 +698,7 @@ public class LocalRunner implements AutoCloseable {
instanceConfig
.setExposePulsarAdminClientEnabled(functionConfig.getExposePulsarAdminClientEnabled());
}
+ instanceConfig.setPulsarClientMemoryLimit(functionConfig.getPulsarClientMemoryLimit());
}
RuntimeSpawner runtimeSpawner = new RuntimeSpawner(
diff --git a/pulsar-functions/proto/src/main/proto/Function.proto b/pulsar-functions/proto/src/main/proto/Function.proto
index 8742b4740ac7..da545179ed9e 100644
--- a/pulsar-functions/proto/src/main/proto/Function.proto
+++ b/pulsar-functions/proto/src/main/proto/Function.proto
@@ -106,6 +106,15 @@ message FunctionDetails {
bool retainKeyOrdering = 22;
SubscriptionPosition subscriptionPosition = 23;
ClientApi clientApi = 24;
+ MemoryLimit pulsarClientMemoryLimit = 25;
+}
+
+// Memory limit of the Pulsar client used by a function instance. A 0 value means unset.
+message MemoryLimit {
+ // The max memory in bytes the Pulsar client can use
+ int64 limitInBytes = 1;
+ // The max memory the Pulsar client can use, as a percentage of the JVM max direct memory
+ double percentOfMaxDirectMemory = 2;
}
message ConsumerSpec {
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
index 9eae1997e51d..194deff1050d 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
@@ -29,6 +29,7 @@ import io.prometheus.client.exporter.HTTPServer;
import java.lang.reflect.Type;
import java.net.InetSocketAddress;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
@@ -37,6 +38,7 @@ import net.bytebuddy.description.type.TypeDefinition;
import net.bytebuddy.dynamic.ClassFileLocator;
import net.bytebuddy.pool.TypePool;
import org.apache.commons.lang3.StringUtils;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.functions.WindowConfig;
import org.apache.pulsar.common.nar.NarClassLoader;
import org.apache.pulsar.common.util.Reflections;
@@ -161,6 +163,15 @@ public class JavaInstanceStarter implements AutoCloseable {
required = false)
public Boolean ignoreUnknownConfigFields = false;
+ @Option(names = "--pulsar_client_memory_limit_bytes",
+ description = "The max memory in bytes the Pulsar client can use",
+ required = false)
+ public Long pulsarClientMemoryLimitBytes;
+
+ @Option(names = "--pulsar_client_memory_limit_percent",
+ description = "The max memory the Pulsar client can use, as a percentage of the JVM max direct memory",
+ required = false)
+ public Double pulsarClientMemoryLimitPercent;
private Server server;
private RuntimeSpawner runtimeSpawner;
@@ -189,6 +200,11 @@ public class JavaInstanceStarter implements AutoCloseable {
instanceConfig.setMaxPendingAsyncRequests(maxPendingAsyncRequests);
instanceConfig.setExposePulsarAdminClientEnabled(exposePulsarAdminClientEnabled);
instanceConfig.setIgnoreUnknownConfigFields(ignoreUnknownConfigFields);
+ if (pulsarClientMemoryLimitBytes != null || pulsarClientMemoryLimitPercent != null) {
+ instanceConfig.setPulsarClientMemoryLimit(new MemoryLimit()
+ .setLimitInBytes(pulsarClientMemoryLimitBytes)
+ .setPercentOfMaxDirectMemory(pulsarClientMemoryLimitPercent));
+ }
if (functionDetailsJsonString.charAt(0) == '\'') {
functionDetailsJsonString = functionDetailsJsonString.substring(1);
}
@@ -244,7 +260,8 @@ public class JavaInstanceStarter implements AutoCloseable {
.tlsHostnameVerificationEnable(isTrue(tlsHostNameVerificationEnabled))
.tlsTrustCertsFilePath(tlsTrustCertFilePath).build(),
secretsProvider, collectorRegistry, narExtractionDirectory, rootClassLoader,
- exposePulsarAdminClientEnabled, webServiceUrl, fnCache);
+ exposePulsarAdminClientEnabled, webServiceUrl, fnCache,
+ Optional.ofNullable(instanceConfig.getPulsarClientMemoryLimit()));
runtimeSpawner = new RuntimeSpawner(
instanceConfig,
jarFile,
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
index 45f687b27ca9..3074c9543fa4 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java
@@ -47,6 +47,7 @@ import lombok.CustomLog;
import org.apache.commons.lang3.JavaVersion;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.util.ObjectMapperFactory;
import org.apache.pulsar.functions.instance.AuthenticationConfig;
import org.apache.pulsar.functions.instance.InstanceConfig;
@@ -507,6 +508,16 @@ public class RuntimeUtils {
if (instanceConfig.isIgnoreUnknownConfigFields()) {
args.add("--ignore_unknown_config_fields");
}
+
+ MemoryLimit clientMemoryLimit = instanceConfig.getPulsarClientMemoryLimit();
+ if (clientMemoryLimit != null && clientMemoryLimit.getLimitInBytes() != null) {
+ args.add("--pulsar_client_memory_limit_bytes");
+ args.add(String.valueOf(clientMemoryLimit.getLimitInBytes()));
+ }
+ if (clientMemoryLimit != null && clientMemoryLimit.getPercentOfMaxDirectMemory() != null) {
+ args.add("--pulsar_client_memory_limit_percent");
+ args.add(String.valueOf(clientMemoryLimit.getPercentOfMaxDirectMemory()));
+ }
}
// state storage configs
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java
index 032fda7b00e4..93765b175e47 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java
@@ -19,6 +19,7 @@
package org.apache.pulsar.functions.runtime.thread;
import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.pulsar.common.util.DirectMemoryUtils.jvmMaxDirectMemoryPercentage;
import java.util.Optional;
import lombok.CustomLog;
import lombok.Getter;
@@ -27,7 +28,7 @@ import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.api.ClientBuilder;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.common.util.DirectMemoryUtils;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.util.Reflections;
import org.apache.pulsar.functions.auth.FunctionAuthProvider;
import org.apache.pulsar.functions.instance.AuthenticationConfig;
@@ -84,8 +85,8 @@ public class ThreadRuntimeFactory implements RuntimeFactory {
AuthenticationConfig authConfig, SecretsProvider secretsProvider,
FunctionCollectorRegistry collectorRegistry, String narExtractionDirectory,
ClassLoader rootClassLoader, boolean exposePulsarAdminClientEnabled,
- String pulsarWebServiceUrl) throws Exception {
- initialize(threadGroupName, Optional.empty(), pulsarServiceUrl, authConfig,
+ String pulsarWebServiceUrl, Optional<MemoryLimit> memoryLimit) throws Exception {
+ initialize(threadGroupName, memoryLimit, pulsarServiceUrl, authConfig,
stateStorageImplClass, storageServiceUrl, null, secretsProvider, collectorRegistry,
narExtractionDirectory,
rootClassLoader, exposePulsarAdminClientEnabled, pulsarWebServiceUrl, Optional.empty(),
@@ -98,15 +99,16 @@ public class ThreadRuntimeFactory implements RuntimeFactory {
AuthenticationConfig authConfig, SecretsProvider secretsProvider,
FunctionCollectorRegistry collectorRegistry, String narExtractionDirectory,
ClassLoader rootClassLoader, boolean exposePulsarAdminClientEnabled,
- String pulsarWebServiceUrl, FunctionCacheManager fnCache) throws Exception {
- initialize(threadGroupName, Optional.empty(), pulsarServiceUrl, authConfig,
+ String pulsarWebServiceUrl, FunctionCacheManager fnCache,
+ Optional<MemoryLimit> memoryLimit) throws Exception {
+ initialize(threadGroupName, memoryLimit, pulsarServiceUrl, authConfig,
stateStorageImplClass, storageServiceUrl, null, secretsProvider, collectorRegistry,
narExtractionDirectory,
rootClassLoader, exposePulsarAdminClientEnabled, pulsarWebServiceUrl, Optional.empty(),
Optional.empty(), fnCache);
}
- private void initialize(String threadGroupName, Optional<ThreadRuntimeFactoryConfig.MemoryLimit> memoryLimit,
+ private void initialize(String threadGroupName, Optional<MemoryLimit> memoryLimit,
String pulsarServiceUrl, AuthenticationConfig authConfig, String stateStorageImplClass,
String storageServiceUrl,
SecretsProviderConfigurator secretsProviderConfigurator, SecretsProvider secretsProvider,
@@ -145,10 +147,10 @@ public class ThreadRuntimeFactory implements RuntimeFactory {
this.functionsManager = functionsManager;
}
- private Optional<Long> calculateClientMemoryLimit(Optional<ThreadRuntimeFactoryConfig.MemoryLimit> memoryLimit) {
+ private Optional<Long> calculateClientMemoryLimit(Optional<MemoryLimit> memoryLimit) {
if (memoryLimit.isPresent()) {
- Long absolute = memoryLimit.get().getAbsoluteValue();
+ Long absolute = memoryLimit.get().getLimitInBytes();
Double percentOfDirectMem = memoryLimit.get().getPercentOfMaxDirectMemory();
if (absolute != null) {
checkArgument(absolute > 0, "Absolute memory limit for Pulsar client has to be positive");
@@ -159,7 +161,7 @@ public class ThreadRuntimeFactory implements RuntimeFactory {
}
if (absolute != null && percentOfDirectMem != null) {
- return Optional.of(Math.min(absolute, getBytesPercentDirectMem(percentOfDirectMem)));
+ return Optional.of(Math.min(absolute, jvmMaxDirectMemoryPercentage(percentOfDirectMem)));
}
if (absolute != null) {
@@ -167,14 +169,21 @@ public class ThreadRuntimeFactory implements RuntimeFactory {
}
if (percentOfDirectMem != null) {
- return Optional.of(getBytesPercentDirectMem(percentOfDirectMem));
+ return Optional.of(jvmMaxDirectMemoryPercentage(percentOfDirectMem));
}
}
return Optional.empty();
}
- private long getBytesPercentDirectMem(double percent) {
- return (long) (DirectMemoryUtils.jvmMaxDirectMemory() * (percent / 100));
+ private static Optional<MemoryLimit> getPulsarClientMemoryLimit(ThreadRuntimeFactoryConfig factoryConfig,
+ WorkerConfig workerConfig) {
+ ThreadRuntimeFactoryConfig.MemoryLimit memoryLimit = factoryConfig.getPulsarClientMemoryLimit();
+ if (memoryLimit == null) {
+ return Optional.ofNullable(workerConfig.getPulsarClientMemoryLimit());
+ }
+ return Optional.of(new MemoryLimit()
+ .setLimitInBytes(memoryLimit.getAbsoluteValue())
+ .setPercentOfMaxDirectMemory(memoryLimit.getPercentOfMaxDirectMemory()));
}
@@ -188,7 +197,7 @@ public class ThreadRuntimeFactory implements RuntimeFactory {
ThreadRuntimeFactoryConfig factoryConfig = RuntimeUtils.getRuntimeFunctionConfig(
workerConfig.getFunctionRuntimeFactoryConfigs(), ThreadRuntimeFactoryConfig.class);
- initialize(factoryConfig.getThreadGroupName(), Optional.ofNullable(factoryConfig.getPulsarClientMemoryLimit()),
+ initialize(factoryConfig.getThreadGroupName(), getPulsarClientMemoryLimit(factoryConfig, workerConfig),
workerConfig.getPulsarServiceUrl(), authenticationConfig,
workerConfig.getStateStorageProviderImplementation(),
workerConfig.getStateStorageServiceUrl(), secretsProviderConfigurator, null,
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
index b4adbb005ec9..66d97108a479 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
@@ -48,6 +48,7 @@ import org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider;
import org.apache.pulsar.common.configuration.Category;
import org.apache.pulsar.common.configuration.FieldContext;
import org.apache.pulsar.common.configuration.PulsarConfiguration;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.functions.Resources;
import org.apache.pulsar.common.nar.NarClassLoader;
import org.apache.pulsar.common.sasl.SaslConstants;
@@ -876,6 +877,15 @@ public class WorkerConfig implements Serializable, PulsarConfiguration {
)
private int maxPendingAsyncRequests = 1000;
+ @FieldContext(
+ category = CATEGORY_FUNC_RUNTIME_MNG,
+ doc = "Default memory limit of the Pulsar client used by each Java function instance, for functions"
+ + " that don't set their own. If both `limitInBytes` and `percentOfMaxDirectMemory` are set,"
+ + " the smaller of the two limits is used. The thread runtime uses it when"
+ + " `functionRuntimeFactoryConfigs.pulsarClientMemoryLimit` is not set."
+ )
+ private MemoryLimit pulsarClientMemoryLimit;
+
@FieldContext(
doc = "Whether to forward the source message properties to the output message"
)
diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java
index 4c8214e6e6a6..0dbc12a24bf8 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java
@@ -27,6 +27,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import lombok.CustomLog;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.functions.instance.AuthenticationConfig;
import org.apache.pulsar.functions.instance.InstanceConfig;
import org.apache.pulsar.functions.proto.FunctionDetails;
@@ -38,6 +39,7 @@ import org.jose4j.json.internal.json_simple.JSONObject;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
+import picocli.CommandLine;
@CustomLog
public class RuntimeUtilsTest {
@@ -239,6 +241,46 @@ public class RuntimeUtilsTest {
assertTrue(indexAdditionalArguments < indexJavaClass);
}
+ @DataProvider(name = "pulsarClientMemoryLimit")
+ public Object[][] pulsarClientMemoryLimit() {
+ return new Object[][] {
+ {null},
+ {new MemoryLimit().setLimitInBytes(64L * 1024 * 1024)},
+ {new MemoryLimit().setPercentOfMaxDirectMemory(25.0)},
+ {new MemoryLimit().setLimitInBytes(64L * 1024 * 1024).setPercentOfMaxDirectMemory(25.0)},
+ };
+ }
+
+ @Test(dataProvider = "pulsarClientMemoryLimit")
+ public void testPulsarClientMemoryLimitReachesJavaInstance(MemoryLimit memoryLimit) throws Exception {
+ InstanceConfig instanceConfig = new InstanceConfig();
+ instanceConfig.setClusterName("kluster");
+ instanceConfig.setInstanceId(3000);
+ instanceConfig.setFunctionId("func-7734");
+ instanceConfig.setFunctionVersion("1.0.0");
+ instanceConfig.setMaxBufferedTuples(5);
+ instanceConfig.setPort(1337);
+ instanceConfig.setFunctionDetails(new FunctionDetails());
+ instanceConfig.setPulsarClientMemoryLimit(memoryLimit);
+
+ List<String> cmd = RuntimeUtils.getCmd(instanceConfig, "instanceFile",
+ null, "logDirectory", "originalCodeFileName", null, "pulsarServiceUrl",
+ null, AuthenticationConfig.builder().build(), "0", 23, 1234L, "logConfigFile",
+ null, null, false, null, null, "narExtractionDirectory", "functionInstanceClassPath",
+ false, "");
+
+ // the arguments after the main class are the ones JavaInstanceStarter parses in the instance process
+ int indexJavaClass = cmd.indexOf("org.apache.pulsar.functions.instance.JavaInstanceMain");
+ JavaInstanceStarter javaInstanceStarter = new JavaInstanceStarter();
+ new CommandLine(javaInstanceStarter).parseArgs(
+ cmd.subList(indexJavaClass + 1, cmd.size()).toArray(new String[0]));
+
+ assertEquals(javaInstanceStarter.pulsarClientMemoryLimitBytes,
+ memoryLimit != null ? memoryLimit.getLimitInBytes() : null);
+ assertEquals(javaInstanceStarter.pulsarClientMemoryLimitPercent,
+ memoryLimit != null ? memoryLimit.getPercentOfMaxDirectMemory() : null);
+ }
+
@Test
public void testSanitizeFileName() {
assertEquals(RuntimeUtils.sanitizeFileName("file(1).jar"), "file_1_.jar");
diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactoryTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactoryTest.java
index 044b291cfd91..ab089db7202a 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactoryTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactoryTest.java
@@ -29,6 +29,7 @@ import lombok.CustomLog;
import org.apache.pulsar.client.api.ClientBuilder;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.SizeUnit;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.util.DirectMemoryUtils;
import org.apache.pulsar.common.util.ObjectMapperFactory;
import org.apache.pulsar.functions.instance.AuthenticationConfig;
@@ -101,8 +102,37 @@ public class ThreadRuntimeFactoryTest {
}
- @SuppressWarnings("unchecked")
+ @Test
+ public void testWorkerMemoryLimitUsedWhenFactoryLimitNotSet() throws Exception {
+ ClientBuilder clientBuilder = initWithMemoryLimits(null, new MemoryLimit().setPercentOfMaxDirectMemory(50.0));
+ Mockito.verify(clientBuilder, Mockito.times(1))
+ .memoryLimit(Mockito.eq(JVM_MAX_DIRECT_MEMORY / 2), Mockito.eq(SizeUnit.BYTES));
+ }
+
+ @Test
+ public void testFactoryMemoryLimitOverridesWorkerMemoryLimit() throws Exception {
+ ClientBuilder clientBuilder = initWithMemoryLimits(
+ new ThreadRuntimeFactoryConfig.MemoryLimit().setAbsoluteValue(JVM_MAX_DIRECT_MEMORY / 4),
+ new MemoryLimit().setPercentOfMaxDirectMemory(50.0));
+ Mockito.verify(clientBuilder, Mockito.times(1))
+ .memoryLimit(Mockito.eq(JVM_MAX_DIRECT_MEMORY / 4), Mockito.eq(SizeUnit.BYTES));
+ }
+
private ClientBuilder testMemoryLimit(Long absolute, Double percent) throws Exception {
+ ThreadRuntimeFactoryConfig.MemoryLimit memoryLimit = new ThreadRuntimeFactoryConfig.MemoryLimit();
+ if (percent != null) {
+ memoryLimit.setPercentOfMaxDirectMemory(percent);
+ }
+
+ if (absolute != null) {
+ memoryLimit.setAbsoluteValue(absolute);
+ }
+ return initWithMemoryLimits(memoryLimit, null);
+ }
+
+ @SuppressWarnings("unchecked")
+ private ClientBuilder initWithMemoryLimits(ThreadRuntimeFactoryConfig.MemoryLimit factoryMemoryLimit,
+ MemoryLimit workerMemoryLimit) throws Exception {
try (MockedStatic<PulsarClient> mockedPulsarClient = mockStatic(PulsarClient.class)) {
ClientBuilder clientBuilder = Mockito.mock(ClientBuilder.class);
mockedPulsarClient.when(PulsarClient::builder).thenAnswer(i -> clientBuilder);
@@ -111,21 +141,14 @@ public class ThreadRuntimeFactoryTest {
ThreadRuntimeFactoryConfig threadRuntimeFactoryConfig = new ThreadRuntimeFactoryConfig();
threadRuntimeFactoryConfig.setThreadGroupName("foo");
- ThreadRuntimeFactoryConfig.MemoryLimit memoryLimit = new ThreadRuntimeFactoryConfig.MemoryLimit();
- if (percent != null) {
- memoryLimit.setPercentOfMaxDirectMemory(percent);
- }
-
- if (absolute != null) {
- memoryLimit.setAbsoluteValue(absolute);
- }
- threadRuntimeFactoryConfig.setPulsarClientMemoryLimit(memoryLimit);
+ threadRuntimeFactoryConfig.setPulsarClientMemoryLimit(factoryMemoryLimit);
WorkerConfig workerConfig = new WorkerConfig();
workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName());
workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getMapper().getObjectMapper()
.convertValue(threadRuntimeFactoryConfig, Map.class));
workerConfig.setPulsarServiceUrl("pulsar://broker.pulsar:6650");
+ workerConfig.setPulsarClientMemoryLimit(workerMemoryLimit);
ThreadRuntimeFactory threadRuntimeFactory = new ThreadRuntimeFactory();
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
index 55bc1b6c0b73..095ea904d00f 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
@@ -48,6 +48,7 @@ import org.apache.pulsar.client.api.SubscriptionInitialPosition;
import org.apache.pulsar.common.functions.ConsumerConfig;
import org.apache.pulsar.common.functions.FunctionConfig;
import org.apache.pulsar.common.functions.FunctionDefinition;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.functions.ProducerConfig;
import org.apache.pulsar.common.functions.Resources;
import org.apache.pulsar.common.functions.WindowConfig;
@@ -378,6 +379,16 @@ public class FunctionConfigUtils {
String builtin = functionConfig.getJar().replaceFirst("^builtin://", "");
functionDetails.setBuiltin(builtin);
}
+ MemoryLimit clientMemoryLimit = functionConfig.getPulsarClientMemoryLimit();
+ if (clientMemoryLimit != null) {
+ org.apache.pulsar.functions.proto.MemoryLimit memoryLimit = functionDetails.setPulsarClientMemoryLimit();
+ if (clientMemoryLimit.getLimitInBytes() != null) {
+ memoryLimit.setLimitInBytes(clientMemoryLimit.getLimitInBytes());
+ }
+ if (clientMemoryLimit.getPercentOfMaxDirectMemory() != null) {
+ memoryLimit.setPercentOfMaxDirectMemory(clientMemoryLimit.getPercentOfMaxDirectMemory());
+ }
+ }
functionDetails.setClientApi(FunctionCommon.convertClientApi(functionConfig.getClientApi()));
ClientApiResolver.resolve(functionDetails);
@@ -401,6 +412,21 @@ public class FunctionConfigUtils {
return functionDetails;
}
+ /**
+ * Convert the proto memory limit, where a 0 value means unset, to the config one.
+ */
+ public static MemoryLimit convertPulsarClientMemoryLimit(
+ org.apache.pulsar.functions.proto.MemoryLimit memoryLimit) {
+ MemoryLimit result = new MemoryLimit();
+ if (memoryLimit.getLimitInBytes() != 0) {
+ result.setLimitInBytes(memoryLimit.getLimitInBytes());
+ }
+ if (memoryLimit.getPercentOfMaxDirectMemory() != 0) {
+ result.setPercentOfMaxDirectMemory(memoryLimit.getPercentOfMaxDirectMemory());
+ }
+ return result;
+ }
+
@SuppressWarnings("deprecation")
public static FunctionConfig convertFromDetails(FunctionDetails functionDetails) {
functionDetails = validateFunctionDetails(functionDetails);
@@ -528,6 +554,10 @@ public class FunctionConfigUtils {
if (!isEmpty(functionDetails.getCustomRuntimeOptions())) {
functionConfig.setCustomRuntimeOptions(functionDetails.getCustomRuntimeOptions());
}
+ if (functionDetails.hasPulsarClientMemoryLimit()) {
+ functionConfig.setPulsarClientMemoryLimit(
+ convertPulsarClientMemoryLimit(functionDetails.getPulsarClientMemoryLimit()));
+ }
return functionConfig;
}
@@ -867,6 +897,18 @@ public class FunctionConfigUtils {
ResourceConfigUtils.validate(functionConfig.getResources());
}
+ MemoryLimit clientMemoryLimit = functionConfig.getPulsarClientMemoryLimit();
+ if (clientMemoryLimit != null) {
+ if (clientMemoryLimit.getLimitInBytes() != null && clientMemoryLimit.getLimitInBytes() <= 0) {
+ throw new IllegalArgumentException("Pulsar client memory limit in bytes must be a positive number");
+ }
+ Double percent = clientMemoryLimit.getPercentOfMaxDirectMemory();
+ if (percent != null && (percent <= 0 || percent > 100)) {
+ throw new IllegalArgumentException(
+ "Pulsar client memory limit percent of max direct memory must be in (0, 100]");
+ }
+ }
+
if (functionConfig.getTimeoutMs() != null && functionConfig.getTimeoutMs() <= 0) {
throw new IllegalArgumentException("Function timeout must be a positive number");
}
@@ -1141,6 +1183,9 @@ public class FunctionConfigUtils {
if (newConfig.getProducerConfig() != null) {
mergedConfig.setProducerConfig(newConfig.getProducerConfig());
}
+ if (newConfig.getPulsarClientMemoryLimit() != null) {
+ mergedConfig.setPulsarClientMemoryLimit(newConfig.getPulsarClientMemoryLimit());
+ }
return mergedConfig;
}
}
diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java
index eec410c062e4..4e54177c068b 100644
--- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java
+++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java
@@ -46,6 +46,7 @@ import org.apache.pulsar.client.impl.schema.JSONSchema;
import org.apache.pulsar.common.functions.ConsumerConfig;
import org.apache.pulsar.common.functions.CryptoConfig;
import org.apache.pulsar.common.functions.FunctionConfig;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.functions.ProducerConfig;
import org.apache.pulsar.common.functions.Resources;
import org.apache.pulsar.common.functions.WindowConfig;
@@ -62,6 +63,7 @@ import org.apache.pulsar.functions.proto.RetryDetails;
import org.apache.pulsar.functions.proto.SinkSpec;
import org.apache.pulsar.functions.proto.SourceSpec;
import org.apache.pulsar.functions.proto.SubscriptionType;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
/**
@@ -139,6 +141,65 @@ public class FunctionConfigUtilsTest {
);
}
+ @DataProvider(name = "pulsarClientMemoryLimit")
+ public Object[][] pulsarClientMemoryLimit() {
+ return new Object[][] {
+ {new MemoryLimit().setLimitInBytes(1024L)},
+ {new MemoryLimit().setPercentOfMaxDirectMemory(25.0)},
+ {new MemoryLimit().setLimitInBytes(1024L).setPercentOfMaxDirectMemory(25.0)},
+ };
+ }
+
+ @Test(dataProvider = "pulsarClientMemoryLimit")
+ public void testConvertPulsarClientMemoryLimit(MemoryLimit memoryLimit) {
+ FunctionConfig functionConfig = createFunctionConfig();
+ functionConfig.setPulsarClientMemoryLimit(memoryLimit);
+
+ FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig);
+ assertEquals(FunctionConfigUtils.convertFromDetails(functionDetails).getPulsarClientMemoryLimit(),
+ memoryLimit);
+ }
+
+ @Test
+ public void testConvertWithoutPulsarClientMemoryLimit() {
+ FunctionDetails functionDetails = FunctionConfigUtils.convert(createFunctionConfig());
+ assertFalse(functionDetails.hasPulsarClientMemoryLimit());
+ assertNull(FunctionConfigUtils.convertFromDetails(functionDetails).getPulsarClientMemoryLimit());
+ }
+
+ @DataProvider(name = "invalidPulsarClientMemoryLimit")
+ public Object[][] invalidPulsarClientMemoryLimit() {
+ return new Object[][] {
+ {new MemoryLimit().setLimitInBytes(0L)},
+ {new MemoryLimit().setLimitInBytes(-1L)},
+ {new MemoryLimit().setPercentOfMaxDirectMemory(0.0)},
+ {new MemoryLimit().setPercentOfMaxDirectMemory(100.1)},
+ };
+ }
+
+ @Test(dataProvider = "invalidPulsarClientMemoryLimit", expectedExceptions = IllegalArgumentException.class,
+ expectedExceptionsMessageRegExp = "Pulsar client memory limit .*")
+ public void testValidateInvalidPulsarClientMemoryLimit(MemoryLimit memoryLimit) {
+ FunctionConfig functionConfig = createFunctionConfig();
+ functionConfig.setPulsarClientMemoryLimit(memoryLimit);
+ FunctionConfigUtils.doCommonChecks(functionConfig);
+ }
+
+ @Test
+ public void testMergePulsarClientMemoryLimit() {
+ FunctionConfig functionConfig = createFunctionConfig();
+ functionConfig.setPulsarClientMemoryLimit(new MemoryLimit().setLimitInBytes(1024L));
+
+ // an update without a limit keeps the existing one
+ FunctionConfig mergedConfig = FunctionConfigUtils.validateUpdate(functionConfig, createFunctionConfig());
+ assertEquals(mergedConfig.getPulsarClientMemoryLimit(), new MemoryLimit().setLimitInBytes(1024L));
+
+ FunctionConfig newFunctionConfig = createFunctionConfig();
+ newFunctionConfig.setPulsarClientMemoryLimit(new MemoryLimit().setPercentOfMaxDirectMemory(25.0));
+ mergedConfig = FunctionConfigUtils.validateUpdate(functionConfig, newFunctionConfig);
+ assertEquals(mergedConfig.getPulsarClientMemoryLimit(), new MemoryLimit().setPercentOfMaxDirectMemory(25.0));
+ }
+
@SuppressWarnings("deprecation")
@Test
public void testConvertWindow() {
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
index 8023aa349e2f..e5fe9f325841 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
@@ -73,6 +73,7 @@ import org.apache.pulsar.functions.runtime.RuntimeSpawner;
import org.apache.pulsar.functions.utils.Actions;
import org.apache.pulsar.functions.utils.ClientApiResolver;
import org.apache.pulsar.functions.utils.FunctionCommon;
+import org.apache.pulsar.functions.utils.FunctionConfigUtils;
import org.apache.pulsar.functions.utils.SourceConfigUtils;
import org.apache.pulsar.functions.utils.ValidatableFunctionPackage;
import org.apache.pulsar.functions.utils.io.Connector;
@@ -237,6 +238,9 @@ public class FunctionActioner {
instanceConfig.setAdditionalJavaRuntimeArguments(workerConfig.getAdditionalJavaRuntimeArguments());
}
instanceConfig.setIgnoreUnknownConfigFields(workerConfig.isIgnoreUnknownConfigFields());
+ instanceConfig.setPulsarClientMemoryLimit(functionDetails.hasPulsarClientMemoryLimit()
+ ? FunctionConfigUtils.convertPulsarClientMemoryLimit(functionDetails.getPulsarClientMemoryLimit())
+ : workerConfig.getPulsarClientMemoryLimit());
return instanceConfig;
}
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java
index 5526c1e621cc..a55e64953d6b 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java
@@ -36,8 +36,11 @@ import java.util.Optional;
import org.apache.distributedlog.api.namespace.Namespace;
import org.apache.pulsar.client.admin.Packages;
import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.common.functions.MemoryLimit;
import org.apache.pulsar.common.util.ObjectMapperFactory;
import org.apache.pulsar.functions.auth.FunctionAuthProvider;
+import org.apache.pulsar.functions.instance.InstanceConfig;
+import org.apache.pulsar.functions.proto.FunctionDetails;
import org.apache.pulsar.functions.proto.FunctionMetaData;
import org.apache.pulsar.functions.proto.Instance;
import org.apache.pulsar.functions.runtime.Runtime;
@@ -270,4 +273,28 @@ public class FunctionActionerTest {
verify(runtime, times(1)).start();
}
+ @Test
+ public void testCreateInstanceConfigPulsarClientMemoryLimit() {
+ WorkerConfig workerConfig = new WorkerConfig();
+ @SuppressWarnings("resource")
+ FunctionActioner actioner = new FunctionActioner(workerConfig, mock(RuntimeFactory.class),
+ mock(Namespace.class), mock(ConnectorsManager.class), mock(FunctionsManager.class),
+ mock(PulsarAdmin.class), mock(PackageUrlValidator.class));
+
+ // neither the worker nor the function sets a limit
+ InstanceConfig instanceConfig = actioner.createInstanceConfig(new FunctionDetails(), null, 0, "c");
+ assertNull(instanceConfig.getPulsarClientMemoryLimit());
+
+ // the worker default applies to a function without its own limit
+ workerConfig.setPulsarClientMemoryLimit(new MemoryLimit().setPercentOfMaxDirectMemory(50.0));
+ instanceConfig = actioner.createInstanceConfig(new FunctionDetails(), null, 0, "c");
+ assertEquals(instanceConfig.getPulsarClientMemoryLimit(),
+ new MemoryLimit().setPercentOfMaxDirectMemory(50.0));
+
+ // the function's own limit overrides the worker default
+ FunctionDetails functionDetails = new FunctionDetails();
+ functionDetails.setPulsarClientMemoryLimit().setLimitInBytes(1024);
+ instanceConfig = actioner.createInstanceConfig(functionDetails, null, 0, "c");
+ assertEquals(instanceConfig.getPulsarClientMemoryLimit(), new MemoryLimit().setLimitInBytes(1024L));
+ }
}