You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by si...@apache.org on 2018/04/12 09:40:14 UTC
[incubator-pulsar] branch master updated: renaming/changing
FunctionConfig (#1552)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 3fbb4c9 renaming/changing FunctionConfig (#1552)
3fbb4c9 is described below
commit 3fbb4c930f76c2e5fef1def8d23831911c8f78e9
Author: Boyang Jerry Peng <je...@gmail.com>
AuthorDate: Thu Apr 12 02:40:10 2018 -0700
renaming/changing FunctionConfig (#1552)
* renaming/changing FunctionConfig
* adding missing license header
* moving FunctionConfig to different module
---
.../pulsar/broker/admin/impl/FunctionsBase.java | 8 +-
.../org/apache/pulsar/client/admin/Functions.java | 12 +-
.../client/admin/internal/FunctionsImpl.java | 26 +-
.../apache/pulsar/admin/cli/CmdFunctionsTest.java | 16 +-
.../org/apache/pulsar/admin/cli/CmdFunctions.java | 262 +++++++----
.../pulsar/functions/instance/ContextImpl.java | 14 +-
.../pulsar/functions/instance/InstanceConfig.java | 4 +-
.../pulsar/functions/instance/JavaInstance.java | 2 +-
.../functions/instance/JavaInstanceRunnable.java | 106 ++---
.../instance/processors/AtLeastOnceProcessor.java | 8 +-
.../instance/processors/AtMostOnceProcessor.java | 10 +-
.../processors/EffectivelyOnceProcessor.java | 12 +-
.../instance/processors/MessageProcessor.java | 18 +-
.../instance/processors/MessageProcessorBase.java | 22 +-
.../instance/src/main/python/Function_pb2.py | 248 +++++------
.../instance/src/main/python/python_instance.py | 42 +-
.../src/main/python/python_instance_main.py | 28 +-
.../instance/JavaInstanceRunnableProcessTest.java | 484 ++++++++++-----------
.../instance/JavaInstanceRunnableTest.java | 12 +-
.../functions/instance/JavaInstanceTest.java | 10 +-
.../pulsar/functions/metrics/MetricsSink.java | 7 +-
.../pulsar/functions/metrics/sink/FileSink.java | 6 +-
.../functions/metrics/sink/PrometheusSink.java | 12 +-
.../metrics/sink/PrometheusSinkTests.java | 36 +-
.../proto/src/main/proto/Function.proto | 4 +-
...ionConfigTest.java => FunctionDetailsTest.java} | 10 +-
.../pulsar/functions/runtime/JavaInstanceMain.java | 36 +-
.../pulsar/functions/runtime/ProcessRuntime.java | 48 +-
.../functions/runtime/ProcessRuntimeFactory.java | 4 +-
.../pulsar/functions/runtime/RuntimeSpawner.java | 2 +-
.../pulsar/functions/runtime/ThreadRuntime.java | 6 +-
.../functions/runtime/ProcessRuntimeTest.java | 68 +--
.../pulsar/functions/utils/FunctionConfig.java | 74 ++++
...nConfigUtils.java => FunctionDetailsUtils.java} | 49 +--
.../pulsar/functions/worker/FunctionActioner.java | 18 +-
.../functions/worker/FunctionMetaDataManager.java | 42 +-
.../functions/worker/FunctionRuntimeManager.java | 18 +-
.../functions/worker/FunctionsStatsGenerator.java | 6 +-
.../pulsar/functions/worker/MembershipManager.java | 14 +-
.../pulsar/functions/worker/SchedulerManager.java | 2 +-
.../org/apache/pulsar/functions/worker/Utils.java | 6 +-
.../functions/worker/rest/api/FunctionsImpl.java | 74 ++--
.../worker/rest/api/v2/FunctionApiV2Resource.java | 8 +-
.../worker/FunctionMetaDataManagerTest.java | 44 +-
.../worker/FunctionRuntimeManagerTest.java | 26 +-
.../worker/FunctionStatsGeneratorTest.java | 4 +-
.../functions/worker/MembershipManagerTest.java | 25 +-
.../functions/worker/SchedulerManagerTest.java | 30 +-
.../rest/api/v2/FunctionApiV2ResourceTest.java | 62 +--
49 files changed, 1083 insertions(+), 1002 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java
index e1318ba..b53c1b1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java
@@ -57,10 +57,10 @@ public class FunctionsBase extends AdminResource implements Supplier<WorkerServi
final @PathParam("functionName") String functionName,
final @FormDataParam("data") InputStream uploadedInputStream,
final @FormDataParam("data") FormDataContentDisposition fileDetail,
- final @FormDataParam("functionConfig") String functionConfigJson) {
+ final @FormDataParam("functionDetails") String functionDetailsJson) {
return functions.registerFunction(
- tenant, namespace, functionName, uploadedInputStream, fileDetail, functionConfigJson);
+ tenant, namespace, functionName, uploadedInputStream, fileDetail, functionDetailsJson);
}
@@ -72,10 +72,10 @@ public class FunctionsBase extends AdminResource implements Supplier<WorkerServi
final @PathParam("functionName") String functionName,
final @FormDataParam("data") InputStream uploadedInputStream,
final @FormDataParam("data") FormDataContentDisposition fileDetail,
- final @FormDataParam("functionConfig") String functionConfigJson) {
+ final @FormDataParam("functionDetails") String functionDetailsJson) {
return functions.updateFunction(
- tenant, namespace, functionName, uploadedInputStream, fileDetail, functionConfigJson);
+ tenant, namespace, functionName, uploadedInputStream, fileDetail, functionDetailsJson);
}
diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Functions.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Functions.java
index 3808a48..4a3dbe5 100644
--- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Functions.java
+++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Functions.java
@@ -21,7 +21,7 @@ package org.apache.pulsar.client.admin;
import org.apache.pulsar.client.admin.PulsarAdminException.NotAuthorizedException;
import org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException;
import org.apache.pulsar.client.admin.PulsarAdminException.PreconditionFailedException;
-import org.apache.pulsar.functions.shaded.proto.Function.FunctionConfig;
+import org.apache.pulsar.functions.shaded.proto.Function.FunctionDetails;
import org.apache.pulsar.functions.shaded.proto.InstanceCommunication.FunctionStatusList;
import java.util.List;
@@ -73,24 +73,24 @@ public interface Functions {
* @throws PulsarAdminException
* Unexpected error
*/
- FunctionConfig getFunction(String tenant, String namespace, String function) throws PulsarAdminException;
+ FunctionDetails getFunction(String tenant, String namespace, String function) throws PulsarAdminException;
/**
* Create a new function.
*
- * @param functionConfig
+ * @param functionDetails
* the function configuration object
*
* @throws PulsarAdminException
* Unexpected error
*/
- void createFunction(FunctionConfig functionConfig, String fileName) throws PulsarAdminException;
+ void createFunction(FunctionDetails functionDetails, String fileName) throws PulsarAdminException;
/**
* Update the configuration for a function.
* <p>
*
- * @param functionConfig
+ * @param functionDetails
* the function configuration object
*
* @throws NotAuthorizedException
@@ -100,7 +100,7 @@ public interface Functions {
* @throws PulsarAdminException
* Unexpected error
*/
- void updateFunction(FunctionConfig functionConfig, String fileName) throws PulsarAdminException;
+ void updateFunction(FunctionDetails functionDetails, String fileName) throws PulsarAdminException;
/**
* Delete an existing function
diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java
index 175e4f3..eb793cc 100644
--- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java
+++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java
@@ -23,7 +23,7 @@ import org.apache.pulsar.client.admin.Functions;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.api.Authentication;
import org.apache.pulsar.common.policies.data.*;
-import org.apache.pulsar.functions.shaded.proto.Function.FunctionConfig;
+import org.apache.pulsar.functions.shaded.proto.Function.FunctionDetails;
import org.apache.pulsar.functions.shaded.proto.InstanceCommunication.FunctionStatusList;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
@@ -68,16 +68,16 @@ public class FunctionsImpl extends BaseResource implements Functions {
}
@Override
- public FunctionConfig getFunction(String tenant, String namespace, String function) throws PulsarAdminException {
+ public FunctionDetails getFunction(String tenant, String namespace, String function) throws PulsarAdminException {
try {
Response response = request(functions.path(tenant).path(namespace).path(function)).get();
if (!response.getStatusInfo().equals(Response.Status.OK)) {
throw new ClientErrorException(response);
}
String jsonResponse = response.readEntity(String.class);
- FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder();
- mergeJson(jsonResponse, functionConfigBuilder);
- return functionConfigBuilder.build();
+ FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
+ mergeJson(jsonResponse, functionDetailsBuilder);
+ return functionDetailsBuilder.build();
} catch (Exception e) {
throw getApiException(e);
}
@@ -101,16 +101,16 @@ public class FunctionsImpl extends BaseResource implements Functions {
}
@Override
- public void createFunction(FunctionConfig functionConfig, String fileName) throws PulsarAdminException {
+ public void createFunction(FunctionDetails functionDetails, String fileName) throws PulsarAdminException {
try {
final FormDataMultiPart mp = new FormDataMultiPart();
mp.bodyPart(new FileDataBodyPart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM_TYPE));
- mp.bodyPart(new FormDataBodyPart("functionConfig",
- printJson(functionConfig),
+ mp.bodyPart(new FormDataBodyPart("functionDetails",
+ printJson(functionDetails),
MediaType.APPLICATION_JSON_TYPE));
- request(functions.path(functionConfig.getTenant()).path(functionConfig.getNamespace()).path(functionConfig.getName()))
+ request(functions.path(functionDetails.getTenant()).path(functionDetails.getNamespace()).path(functionDetails.getName()))
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA), ErrorData.class);
} catch (Exception e) {
throw getApiException(e);
@@ -128,16 +128,16 @@ public class FunctionsImpl extends BaseResource implements Functions {
}
@Override
- public void updateFunction(FunctionConfig functionConfig, String fileName) throws PulsarAdminException {
+ public void updateFunction(FunctionDetails functionDetails, String fileName) throws PulsarAdminException {
try {
final FormDataMultiPart mp = new FormDataMultiPart();
if (fileName != null) {
mp.bodyPart(new FileDataBodyPart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM_TYPE));
}
- mp.bodyPart(new FormDataBodyPart("functionConfig",
- printJson(functionConfig),
+ mp.bodyPart(new FormDataBodyPart("functionDetails",
+ printJson(functionDetails),
MediaType.APPLICATION_JSON_TYPE));
- request(functions.path(functionConfig.getTenant()).path(functionConfig.getNamespace()).path(functionConfig.getName()))
+ request(functions.path(functionDetails.getTenant()).path(functionDetails.getNamespace()).path(functionDetails.getName()))
.put(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA), ErrorData.class);
} catch (Exception e) {
throw getApiException(e);
diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java
index 5f745d1..83220d0 100644
--- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java
+++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java
@@ -53,7 +53,7 @@ import org.apache.pulsar.client.impl.conf.ClientConfigurationData;
import org.apache.pulsar.functions.api.Context;
import org.apache.pulsar.functions.api.Function;
import org.apache.pulsar.functions.api.utils.DefaultSerDe;
-import org.apache.pulsar.functions.shaded.proto.Function.FunctionConfig;
+import org.apache.pulsar.functions.shaded.proto.Function.FunctionDetails;
import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBuf;
import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBufUtil;
import org.apache.pulsar.functions.utils.Reflections;
@@ -188,7 +188,7 @@ public class CmdFunctionsTest {
assertEquals(inputTopicName, creater.getInputs());
assertEquals(outputTopicName, creater.getOutput());
- verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString());
+ verify(functions, times(1)).createFunction(any(FunctionDetails.class), anyString());
}
@@ -209,7 +209,7 @@ public class CmdFunctionsTest {
CreateFunction creater = cmd.getCreater();
assertEquals("tenant", creater.getFunctionConfig().getTenant());
- verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString());
+ verify(functions, times(1)).createFunction(any(FunctionDetails.class), anyString());
}
@Test
@@ -229,7 +229,7 @@ public class CmdFunctionsTest {
CreateFunction creater = cmd.getCreater();
assertEquals("tenant", creater.getFunctionConfig().getTenant());
assertEquals("namespace", creater.getFunctionConfig().getNamespace());
- verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString());
+ verify(functions, times(1)).createFunction(any(FunctionDetails.class), anyString());
}
@Test
@@ -254,7 +254,7 @@ public class CmdFunctionsTest {
assertEquals(tenant, creater.getFunctionConfig().getTenant());
assertEquals(namespace, creater.getFunctionConfig().getNamespace());
assertEquals(functionName, creater.getFunctionConfig().getName());
- verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString());
+ verify(functions, times(1)).createFunction(any(FunctionDetails.class), anyString());
}
@Test
@@ -273,7 +273,7 @@ public class CmdFunctionsTest {
CreateFunction creater = cmd.getCreater();
assertEquals("CmdFunctionsTest$DummyFunction", creater.getFunctionConfig().getName());
- verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString());
+ verify(functions, times(1)).createFunction(any(FunctionDetails.class), anyString());
}
@Test
@@ -290,7 +290,7 @@ public class CmdFunctionsTest {
CreateFunction creater = cmd.getCreater();
assertEquals(inputTopicName + "-" + "CmdFunctionsTest$DummyFunction" + "-output", creater.getFunctionConfig().getOutput());
- verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString());
+ verify(functions, times(1)).createFunction(any(FunctionDetails.class), anyString());
}
@Test
@@ -359,7 +359,7 @@ public class CmdFunctionsTest {
assertEquals(inputTopicName, updater.getInputs());
assertEquals(outputTopicName, updater.getOutput());
- verify(functions, times(1)).updateFunction(any(FunctionConfig.class), anyString());
+ verify(functions, times(1)).updateFunction(any(FunctionDetails.class), anyString());
}
@Test
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
index 66b8630..71ba928 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
@@ -32,6 +32,8 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import org.apache.bookkeeper.api.StorageClient;
import org.apache.bookkeeper.api.kv.Table;
import org.apache.bookkeeper.api.kv.result.KeyValue;
@@ -43,6 +45,7 @@ import org.apache.pulsar.client.admin.internal.FunctionsImpl;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.functions.api.Function;
+import org.apache.pulsar.functions.utils.FunctionConfig;
import org.apache.pulsar.functions.api.SerDe;
import org.apache.pulsar.functions.api.utils.DefaultSerDe;
import org.apache.pulsar.functions.instance.InstanceConfig;
@@ -51,8 +54,7 @@ import org.apache.pulsar.functions.runtime.RuntimeSpawner;
import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBuf;
import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBufUtil;
import org.apache.pulsar.functions.shaded.io.netty.buffer.Unpooled;
-import org.apache.pulsar.functions.shaded.proto.Function.FunctionConfig;
-import org.apache.pulsar.functions.utils.FunctionConfigUtils;
+import org.apache.pulsar.functions.shaded.proto.Function.FunctionDetails;
import org.apache.pulsar.functions.utils.Reflections;
import org.apache.pulsar.functions.utils.Utils;
@@ -162,7 +164,7 @@ public class CmdFunctions extends CmdBase {
* Commands that require a function config
*/
@Getter
- abstract class FunctionConfigCommand extends BaseCommand {
+ abstract class FunctionDetailsCommand extends BaseCommand {
@Parameter(names = "--fqfn", description = "The Fully Qualified Function Name (FQFN) for the function")
protected String fqfn;
@Parameter(names = "--tenant", description = "The function's tenant")
@@ -211,103 +213,100 @@ public class CmdFunctions extends CmdBase {
void processArguments() throws Exception {
super.processArguments();
- FunctionConfig.Builder functionConfigBuilder;
-
// Initialize config builder either from a supplied YAML config file or from scratch
if (null != fnConfigFile) {
- functionConfigBuilder = loadConfig(new File(fnConfigFile));
+ functionConfig = loadConfig(new File(fnConfigFile));
} else {
- functionConfigBuilder = FunctionConfig.newBuilder();
+ functionConfig = new FunctionConfig();
}
if (null != fqfn) {
- parseFullyQualifiedFunctionName(fqfn, functionConfigBuilder);
+ parseFullyQualifiedFunctionName(fqfn, functionConfig);
} else {
if (null != tenant) {
- functionConfigBuilder.setTenant(tenant);
+ functionConfig.setTenant(tenant);
}
if (null != namespace) {
- functionConfigBuilder.setNamespace(namespace);
+ functionConfig.setNamespace(namespace);
}
if (null != functionName) {
- functionConfigBuilder.setName(functionName);
+ functionConfig.setName(functionName);
}
}
if (null != inputs) {
- Arrays.asList(inputs.split(",")).forEach(functionConfigBuilder::addInputs);
+ Arrays.asList(inputs.split(",")).forEach(functionConfig.getInputs()::add);
}
if (null != customSerdeInputString) {
Type type = new TypeToken<Map<String, String>>(){}.getType();
Map<String, String> customSerdeInputMap = new Gson().fromJson(customSerdeInputString, type);
- functionConfigBuilder.putAllCustomSerdeInputs(customSerdeInputMap);
+ functionConfig.setCustomSerdeInputs(customSerdeInputMap);
}
if (null != output) {
- functionConfigBuilder.setOutput(output);
+ functionConfig.setOutput(output);
}
if (null != logTopic) {
- functionConfigBuilder.setLogTopic(logTopic);
+ functionConfig.setLogTopic(logTopic);
}
if (null != className) {
- functionConfigBuilder.setClassName(className);
+ functionConfig.setClassName(className);
}
if (null != outputSerdeClassName) {
- functionConfigBuilder.setOutputSerdeClassName(outputSerdeClassName);
+ functionConfig.setOutputSerdeClassName(outputSerdeClassName);
}
if (null != processingGuarantees) {
- functionConfigBuilder.setProcessingGuarantees(processingGuarantees);
+ functionConfig.setProcessingGuarantees(processingGuarantees);
}
if (null != subscriptionType) {
- functionConfigBuilder.setSubscriptionType(subscriptionType);
+ functionConfig.setSubscriptionType(subscriptionType);
}
if (null != userConfigString) {
Type type = new TypeToken<Map<String, String>>(){}.getType();
Map<String, String> userConfigMap = new Gson().fromJson(userConfigString, type);
- functionConfigBuilder.putAllUserConfig(userConfigMap);
+ functionConfig.setUserConfig(userConfigMap);
}
if (null != jarFile) {
- doJavaSubmitChecks(functionConfigBuilder);
- functionConfigBuilder.setRuntime(FunctionConfig.Runtime.JAVA);
+ doJavaSubmitChecks(functionConfig);
+ functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
userCodeFile = jarFile;
} else if (null != pyFile) {
- doPythonSubmitChecks(functionConfigBuilder);
- functionConfigBuilder.setRuntime(FunctionConfig.Runtime.PYTHON);
+ doPythonSubmitChecks(functionConfig);
+ functionConfig.setRuntime(FunctionConfig.Runtime.PYTHON);
userCodeFile = pyFile;
} else {
throw new RuntimeException("Either a Java jar or a Python file needs to be specified for the function");
}
- if (functionConfigBuilder.getInputsCount() == 0 && functionConfigBuilder.getCustomSerdeInputsCount() == 0) {
+ if (functionConfig.getInputs().size() == 0 && functionConfig.getCustomSerdeInputs().size() == 0) {
throw new RuntimeException("No input topic(s) specified for the function");
}
if (parallelism == null) {
- if (functionConfigBuilder.getParallelism() == 0) {
- functionConfigBuilder.setParallelism(1);
+ if (functionConfig.getParallelism() == 0) {
+ functionConfig.setParallelism(1);
}
} else {
int num = Integer.parseInt(parallelism);
if (num <= 0) {
throw new IllegalArgumentException("The parallelism factor (the number of instances) for the function must be positive");
}
- functionConfigBuilder.setParallelism(num);
+ functionConfig.setParallelism(num);
}
- functionConfigBuilder.setAutoAck(true);
- inferMissingArguments(functionConfigBuilder);
- functionConfig = functionConfigBuilder.build();
+ functionConfig.setAutoAck(true);
+ inferMissingArguments(functionConfig);
}
- private void doJavaSubmitChecks(FunctionConfig.Builder functionConfigBuilder) {
+ private void doJavaSubmitChecks(FunctionConfig functionConfig) {
File file = new File(jarFile);
// check if the function class exists in Jar and it implements Function class
- if (!Reflections.classExistsInJar(file, functionConfigBuilder.getClassName())) {
+ if (!Reflections.classExistsInJar(file, functionConfig.getClassName())) {
throw new IllegalArgumentException(String.format("Pulsar function class %s does not exist in jar %s",
- functionConfigBuilder.getClassName(), jarFile));
- } else if (!Reflections.classInJarImplementsIface(file, functionConfigBuilder.getClassName(), Function.class)
- && !Reflections.classInJarImplementsIface(file, functionConfigBuilder.getClassName(), java.util.function.Function.class)) {
+ functionConfig.getClassName(), jarFile));
+ } else if (!Reflections.classInJarImplementsIface(file, functionConfig.getClassName(), Function.class)
+ && !Reflections.classInJarImplementsIface(file, functionConfig.getClassName(), java.util.function.Function.class)) {
throw new IllegalArgumentException(String.format("The Pulsar function class %s in jar %s implements neither org.apache.pulsar.functions.api.Function nor java.util.function.Function",
- functionConfigBuilder.getClassName(), jarFile));
+ functionConfig.getClassName(), jarFile));
}
ClassLoader userJarLoader;
@@ -317,27 +316,27 @@ public class CmdFunctions extends CmdBase {
throw new RuntimeException("Failed to load user jar " + file, e);
}
- Object userClass = Reflections.createInstance(functionConfigBuilder.getClassName(), file);
+ Object userClass = Reflections.createInstance(functionConfig.getClassName(), file);
Class<?>[] typeArgs;
if (userClass instanceof Function) {
Function pulsarFunction = (Function) userClass;
if (pulsarFunction == null) {
throw new IllegalArgumentException(String.format("The Pulsar function class %s could not be instantiated from jar %s",
- functionConfigBuilder.getClassName(), jarFile));
+ functionConfig.getClassName(), jarFile));
}
typeArgs = TypeResolver.resolveRawArguments(Function.class, pulsarFunction.getClass());
} else {
java.util.function.Function function = (java.util.function.Function) userClass;
if (function == null) {
throw new IllegalArgumentException(String.format("The Java util function class %s could not be instantiated from jar %s",
- functionConfigBuilder.getClassName(), jarFile));
+ functionConfig.getClassName(), jarFile));
}
typeArgs = TypeResolver.resolveRawArguments(java.util.function.Function.class, function.getClass());
}
// Check if the Input serialization/deserialization class exists in jar or already loaded and that it
// implements SerDe class
- functionConfigBuilder.getCustomSerdeInputsMap().forEach((topicName, inputSerializer) -> {
+ functionConfig.getCustomSerdeInputs().forEach((topicName, inputSerializer) -> {
if (!Reflections.classExists(inputSerializer)
&& !Reflections.classExistsInJar(new File(jarFile), inputSerializer)) {
throw new IllegalArgumentException(
@@ -382,23 +381,23 @@ public class CmdFunctions extends CmdBase {
}
}
});
- functionConfigBuilder.getInputsList().forEach((topicName) -> {
+ functionConfig.getInputs().forEach((topicName) -> {
if (!DefaultSerDe.IsSupportedType(typeArgs[0])) {
throw new RuntimeException("Default Serializer does not support type " + typeArgs[0]);
}
});
if (!Void.class.equals(typeArgs[1])) {
- if (functionConfigBuilder.getOutputSerdeClassName() == null
- || functionConfigBuilder.getOutputSerdeClassName().isEmpty()
- || functionConfigBuilder.getOutputSerdeClassName().equals(DefaultSerDe.class.getName())) {
+ if (functionConfig.getOutputSerdeClassName() == null
+ || functionConfig.getOutputSerdeClassName().isEmpty()
+ || functionConfig.getOutputSerdeClassName().equals(DefaultSerDe.class.getName())) {
if (!DefaultSerDe.IsSupportedType(typeArgs[1])) {
throw new RuntimeException("Default Serializer does not support type " + typeArgs[1]);
}
} else {
- SerDe serDe = (SerDe) Reflections.createInstance(functionConfigBuilder.getOutputSerdeClassName(), file);
+ SerDe serDe = (SerDe) Reflections.createInstance(functionConfig.getOutputSerdeClassName(), file);
if (serDe == null) {
throw new IllegalArgumentException(String.format("SerDe class %s does not exist in jar %s",
- functionConfigBuilder.getOutputSerdeClassName(), jarFile));
+ functionConfig.getOutputSerdeClassName(), jarFile));
}
Class<?>[] serDeTypes = TypeResolver.resolveRawArguments(SerDe.class, serDe.getClass());
@@ -420,78 +419,78 @@ public class CmdFunctions extends CmdBase {
}
}
- private void doPythonSubmitChecks(FunctionConfig.Builder functionConfigBuilder) {
- if (functionConfigBuilder.getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) {
+ private void doPythonSubmitChecks(FunctionConfig functionConfig) {
+ if (functionConfig.getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) {
throw new RuntimeException("Effectively-once processing guarantees not yet supported in Python");
}
}
- private void inferMissingArguments(FunctionConfig.Builder builder) {
- if (builder.getName() == null || builder.getName().isEmpty()) {
- inferMissingFunctionName(builder);
+ private void inferMissingArguments(FunctionConfig functionConfig) {
+ if (functionConfig.getName() == null || functionConfig.getName().isEmpty()) {
+ inferMissingFunctionName(functionConfig);
}
- if (builder.getTenant() == null || builder.getTenant().isEmpty()) {
- inferMissingTenant(builder);
+ if (functionConfig.getTenant() == null || functionConfig.getTenant().isEmpty()) {
+ inferMissingTenant(functionConfig);
}
- if (builder.getNamespace() == null || builder.getNamespace().isEmpty()) {
- inferMissingNamespace(builder);
+ if (functionConfig.getNamespace() == null || functionConfig.getNamespace().isEmpty()) {
+ inferMissingNamespace(functionConfig);
}
- if (builder.getOutput() == null || builder.getOutput().isEmpty()) {
- inferMissingOutput(builder);
+ if (functionConfig.getOutput() == null || functionConfig.getOutput().isEmpty()) {
+ inferMissingOutput(functionConfig);
}
}
- private void inferMissingFunctionName(FunctionConfig.Builder builder) {
- String [] domains = builder.getClassName().split("\\.");
+ private void inferMissingFunctionName(FunctionConfig functionConfig) {
+ String [] domains = functionConfig.getClassName().split("\\.");
if (domains.length == 0) {
- builder.setName(builder.getClassName());
+ functionConfig.setName(functionConfig.getClassName());
} else {
- builder.setName(domains[domains.length - 1]);
+ functionConfig.setName(domains[domains.length - 1]);
}
}
- private void inferMissingTenant(FunctionConfig.Builder builder) {
+ private void inferMissingTenant(FunctionConfig functionConfig) {
try {
- String inputTopic = getUniqueInput(builder);
- builder.setTenant(TopicName.get(inputTopic).getProperty());
+ String inputTopic = getUniqueInput(functionConfig);
+ functionConfig.setTenant(TopicName.get(inputTopic).getProperty());
} catch (IllegalArgumentException ex) {
throw new RuntimeException("You need to specify a tenant for the function", ex);
}
}
- private void inferMissingNamespace(FunctionConfig.Builder builder) {
+ private void inferMissingNamespace(FunctionConfig functionConfig) {
try {
- String inputTopic = getUniqueInput(builder);
- builder.setNamespace(TopicName.get(inputTopic).getNamespacePortion());
+ String inputTopic = getUniqueInput(functionConfig);
+ functionConfig.setNamespace(TopicName.get(inputTopic).getNamespacePortion());
} catch (IllegalArgumentException ex) {
throw new RuntimeException("You need to specify a namespace for the function");
}
}
- private void inferMissingOutput(FunctionConfig.Builder builder) {
+ private void inferMissingOutput(FunctionConfig functionConfig) {
try {
- String inputTopic = getUniqueInput(builder);
- builder.setOutput(inputTopic + "-" + builder.getName() + "-output");
+ String inputTopic = getUniqueInput(functionConfig);
+ functionConfig.setOutput(inputTopic + "-" + functionConfig.getName() + "-output");
} catch (IllegalArgumentException ex) {
// It might be that we really don't need an output topic
// So we cannot really throw an exception
}
}
- private String getUniqueInput(FunctionConfig.Builder builder) {
- if (builder.getInputsCount() + builder.getCustomSerdeInputsCount() != 1) {
+ private String getUniqueInput(FunctionConfig functionConfig) {
+ if (functionConfig.getInputs().size() + functionConfig.getCustomSerdeInputs().size() != 1) {
throw new IllegalArgumentException();
}
- if (builder.getInputsCount() == 1) {
- return builder.getInputs(0);
+ if (functionConfig.getInputs().size() == 1) {
+ return functionConfig.getInputs().iterator().next();
} else {
- return builder.getCustomSerdeInputsMap().keySet().iterator().next();
+ return functionConfig.getCustomSerdeInputs().keySet().iterator().next();
}
}
}
@Parameters(commandDescription = "Run the Pulsar Function locally (rather than deploying it to the Pulsar cluster)")
- class LocalRunner extends FunctionConfigCommand {
+ class LocalRunner extends FunctionDetailsCommand {
// TODO: this should become bookkeeper url and it should be fetched from pulsar client.
@Parameter(names = "--stateStorageServiceUrl", description = "The URL for the state storage service (by default Apache BookKeeper)")
@@ -518,7 +517,7 @@ public class CmdFunctions extends CmdBase {
List<RuntimeSpawner> spawners = new LinkedList<>();
for (int i = 0; i < functionConfig.getParallelism(); ++i) {
InstanceConfig instanceConfig = new InstanceConfig();
- instanceConfig.setFunctionConfig(convert(functionConfig));
+ instanceConfig.setFunctionDetails(convertProto2(functionConfig));
// TODO: correctly implement function version and id
instanceConfig.setFunctionVersion(UUID.randomUUID().toString());
instanceConfig.setFunctionId(UUID.randomUUID().toString());
@@ -551,13 +550,13 @@ public class CmdFunctions extends CmdBase {
}
@Parameters(commandDescription = "Create a Pulsar Function in cluster mode (i.e. deploy it on a Pulsar cluster)")
- class CreateFunction extends FunctionConfigCommand {
+ class CreateFunction extends FunctionDetailsCommand {
@Override
void runCmd() throws Exception {
if (!areAllRequiredFieldsPresent(functionConfig)) {
throw new RuntimeException("Missing arguments");
}
- admin.functions().createFunction(functionConfig, userCodeFile);
+ admin.functions().createFunction(convert(functionConfig), userCodeFile);
print("Created successfully");
}
}
@@ -592,13 +591,13 @@ public class CmdFunctions extends CmdBase {
}
@Parameters(commandDescription = "Update a Pulsar Function that's been deployed to a Pulsar cluster")
- class UpdateFunction extends FunctionConfigCommand {
+ class UpdateFunction extends FunctionDetailsCommand {
@Override
void runCmd() throws Exception {
if (!areAllRequiredFieldsPresent(functionConfig)) {
throw new RuntimeException("Missing arguments");
}
- admin.functions().updateFunction(functionConfig, userCodeFile);
+ admin.functions().updateFunction(convert(functionConfig), userCodeFile);
print("Updated successfully");
}
}
@@ -750,35 +749,106 @@ public class CmdFunctions extends CmdBase {
return triggerer;
}
- private static FunctionConfig.Builder loadConfig(File file) throws IOException {
- String json = FunctionConfigUtils.convertYamlToJson(file);
- FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder();
- Utils.mergeJson(json, functionConfigBuilder);
- return functionConfigBuilder;
+ private static FunctionConfig loadConfig(File file) throws IOException {
+
+ ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+ return mapper.readValue(file, FunctionConfig.class);
}
public static boolean areAllRequiredFieldsPresent(FunctionConfig functionConfig) {
return functionConfig.getTenant() != null && functionConfig.getNamespace() != null
&& functionConfig.getName() != null && functionConfig.getClassName() != null
- && (functionConfig.getInputsCount() > 0 || functionConfig.getCustomSerdeInputsCount() > 0)
+ && (functionConfig.getInputs().size() > 0 || functionConfig.getCustomSerdeInputs().size() > 0)
&& functionConfig.getParallelism() > 0;
}
+
+ private org.apache.pulsar.functions.proto.Function.FunctionDetails convertProto2(FunctionConfig functionConfig)
+ throws IOException {
+ org.apache.pulsar.functions.proto.Function.FunctionDetails.Builder functionDetailsBuilder = org.apache.pulsar.functions.proto.Function.FunctionDetails.newBuilder();
+ Utils.mergeJson(FunctionsImpl.printJson(convert(functionConfig)), functionDetailsBuilder);
+ return functionDetailsBuilder.build();
+ }
- private org.apache.pulsar.functions.proto.Function.FunctionConfig convert(FunctionConfig functionConfig)
+ private FunctionDetails convert(FunctionConfig functionConfig)
throws IOException {
- org.apache.pulsar.functions.proto.Function.FunctionConfig.Builder functionConfigBuilder = org.apache.pulsar.functions.proto.Function.FunctionConfig.newBuilder();
- Utils.mergeJson(FunctionsImpl.printJson(functionConfig), functionConfigBuilder);
- return functionConfigBuilder.build();
+ FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
+ if (functionConfig.getInputs() != null) {
+ functionDetailsBuilder.setTenant(functionConfig.getTenant());
+ }
+ if (functionConfig.getNamespace() != null) {
+ functionDetailsBuilder.setNamespace(functionConfig.getNamespace());
+ }
+ if (functionConfig.getName() != null) {
+ functionDetailsBuilder.setName(functionConfig.getName());
+ }
+ if (functionConfig.getClassName() != null) {
+ functionDetailsBuilder.setClassName(functionConfig.getClassName());
+ }
+ functionDetailsBuilder.putAllCustomSerdeInputs(functionConfig.getCustomSerdeInputs());
+ if (functionConfig.getOutputSerdeClassName() != null) {
+ functionDetailsBuilder.setOutputSerdeClassName(functionConfig.getOutputSerdeClassName());
+ }
+ if (functionConfig.getOutput() != null) {
+ functionDetailsBuilder.setOutput(functionConfig.getOutput());
+ }
+ if (functionConfig.getLogTopic() != null) {
+ functionDetailsBuilder.setLogTopic(functionConfig.getLogTopic());
+ }
+ if (functionConfig.getProcessingGuarantees() != null) {
+ functionDetailsBuilder.setProcessingGuarantees(
+ convertProcessingGuarantee(functionConfig.getProcessingGuarantees()));
+ }
+ functionDetailsBuilder.putAllUserConfig(functionConfig.getUserConfig());
+ if (functionConfig.getSubscriptionType() != null) {
+ functionDetailsBuilder.setSubscriptionType(
+ convertSubscriptionType(functionConfig.getSubscriptionType()));
+ }
+ if (functionConfig.getRuntime() != null) {
+ functionDetailsBuilder.setRuntime(convertRuntime(functionConfig.getRuntime()));
+ }
+ functionDetailsBuilder.setAutoAck(functionConfig.isAutoAck());
+ functionDetailsBuilder.addAllInputs(functionConfig.getInputs());
+ functionDetailsBuilder.setParallelism(functionConfig.getParallelism());
+ return functionDetailsBuilder.build();
+ }
+
+ private static FunctionDetails.SubscriptionType convertSubscriptionType(
+ FunctionConfig.SubscriptionType subscriptionType) {
+ for (FunctionDetails.SubscriptionType type : FunctionDetails.SubscriptionType.values()) {
+ if (type.name().equals(subscriptionType.name())) {
+ return type;
+ }
+ }
+ throw new RuntimeException("Unrecognized subscription type: " + subscriptionType.name());
+ }
+
+ private static FunctionDetails.ProcessingGuarantees convertProcessingGuarantee(
+ FunctionConfig.ProcessingGuarantees processingGuarantees) {
+ for (FunctionDetails.ProcessingGuarantees type : FunctionDetails.ProcessingGuarantees.values()) {
+ if (type.name().equals(processingGuarantees.name())) {
+ return type;
+ }
+ }
+ throw new RuntimeException("Unrecognized processing guarantee: " + processingGuarantees.name());
+ }
+
+ private static FunctionDetails.Runtime convertRuntime(FunctionConfig.Runtime runtime) {
+ for (FunctionDetails.Runtime type : FunctionDetails.Runtime.values()) {
+ if (type.name().equals(runtime.name())) {
+ return type;
+ }
+ }
+ throw new RuntimeException("Unrecognized runtime: " + runtime.name());
}
- private void parseFullyQualifiedFunctionName(String fqfn, FunctionConfig.Builder functionConfigBuilder) {
+ private void parseFullyQualifiedFunctionName(String fqfn, FunctionConfig functionConfig) {
String[] args = fqfn.split("/");
if (args.length != 3) {
throw new RuntimeException("Fully qualified function names (FQFNs) must be of the form tenant/namespace/name");
} else {
- functionConfigBuilder.setTenant(args[0]);
- functionConfigBuilder.setNamespace(args[1]);
- functionConfigBuilder.setName(args[2]);
+ functionConfig.setTenant(args[0]);
+ functionConfig.setNamespace(args[1]);
+ functionConfig.setName(args[2]);
}
}
}
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java
index 3db2010..4d458f6 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java
@@ -131,27 +131,27 @@ class ContextImpl implements Context {
@Override
public String getOutputTopic() {
- return config.getFunctionConfig().getOutput();
+ return config.getFunctionDetails().getOutput();
}
@Override
public String getOutputSerdeClassName() {
- return config.getFunctionConfig().getOutputSerdeClassName();
+ return config.getFunctionDetails().getOutputSerdeClassName();
}
@Override
public String getTenant() {
- return config.getFunctionConfig().getTenant();
+ return config.getFunctionDetails().getTenant();
}
@Override
public String getNamespace() {
- return config.getFunctionConfig().getNamespace();
+ return config.getFunctionDetails().getNamespace();
}
@Override
public String getFunctionName() {
- return config.getFunctionConfig().getName();
+ return config.getFunctionDetails().getName();
}
@Override
@@ -176,7 +176,7 @@ class ContextImpl implements Context {
@Override
public Optional<String> getUserConfigValue(String key) {
- return Optional.ofNullable(config.getFunctionConfig().getUserConfigOrDefault(key, null));
+ return Optional.ofNullable(config.getFunctionDetails().getUserConfigOrDefault(key, null));
}
@Override
@@ -186,7 +186,7 @@ class ContextImpl implements Context {
@Override
public Map<String, String> getUserConfigMap() {
- return config.getFunctionConfig().getUserConfigMap();
+ return config.getFunctionDetails().getUserConfigMap();
}
@Override
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java
index 3492bb1..5f26e43 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java
@@ -23,7 +23,7 @@ import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails;
/**
* This is the config passed to the Java Instance. Contains all the information
@@ -38,6 +38,6 @@ public class InstanceConfig {
private String instanceId;
private String functionId;
private String functionVersion;
- private FunctionConfig functionConfig;
+ private FunctionDetails functionDetails;
private int maxBufferedTuples;
}
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstance.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstance.java
index 7018bc4..f614ea9 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstance.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstance.java
@@ -50,7 +50,7 @@ public class JavaInstance implements AutoCloseable {
PulsarClient pulsarClient,
Map<String, Consumer> inputConsumers) {
// TODO: cache logger instances by functions?
- Logger instanceLog = LoggerFactory.getLogger("function-" + config.getFunctionConfig().getName());
+ Logger instanceLog = LoggerFactory.getLogger("function-" + config.getFunctionDetails().getName());
this.context = new ContextImpl(config, instanceLog, pulsarClient, clsLoader, inputConsumers);
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
index 008d5a9..d4b6b7f 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
@@ -58,7 +58,7 @@ import org.apache.pulsar.functions.proto.InstanceCommunication;
import org.apache.pulsar.functions.utils.functioncache.FunctionCacheManager;
import org.apache.pulsar.functions.api.SerDe;
import org.apache.pulsar.functions.instance.state.StateContextImpl;
-import org.apache.pulsar.functions.utils.FunctionConfigUtils;
+import org.apache.pulsar.functions.utils.FunctionDetailsUtils;
import org.apache.pulsar.functions.utils.Reflections;
/**
@@ -115,9 +115,9 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable {
this.stateStorageServiceUrl = stateStorageServiceUrl;
this.stats = new FunctionStats();
this.processor = MessageProcessor.create(
- client,
- instanceConfig.getFunctionConfig(),
- queue);
+ client,
+ instanceConfig.getFunctionDetails(),
+ queue);
}
/**
@@ -125,10 +125,10 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable {
*/
JavaInstance setupJavaInstance() throws Exception {
// initialize the thread context
- ThreadContext.put("function", FunctionConfigUtils.getFullyQualifiedName(instanceConfig.getFunctionConfig()));
+ ThreadContext.put("function", FunctionDetailsUtils.getFullyQualifiedName(instanceConfig.getFunctionDetails()));
ThreadContext.put("instance", instanceConfig.getInstanceId());
- log.info("Starting Java Instance {}", instanceConfig.getFunctionConfig().getName());
+ log.info("Starting Java Instance {}", instanceConfig.getFunctionDetails().getName());
// start the function thread
loadJars();
@@ -136,7 +136,7 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable {
ClassLoader clsLoader = Thread.currentThread().getContextClassLoader();
Object object = Reflections.createInstance(
- instanceConfig.getFunctionConfig().getClassName(),
+ instanceConfig.getFunctionDetails().getClassName(),
clsLoader);
if (!(object instanceof Function) && !(object instanceof java.util.function.Function)) {
throw new RuntimeException("User class must either be Function or java.util.Function");
@@ -182,7 +182,7 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable {
log.debug("Received message: {}", msg.getActualMessage().getMessageId());
} catch (InterruptedException ie) {
log.info("Function thread {} is interrupted",
- FunctionConfigUtils.getFullyQualifiedName(instanceConfig.getFunctionConfig()), ie);
+ FunctionDetailsUtils.getFullyQualifiedName(instanceConfig.getFunctionDetails()), ie);
break;
}
@@ -223,12 +223,12 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable {
if (null != stateContext) {
stateContext.flush()
- .thenRun(() -> processResult(msg, result, processAt, doneProcessing))
- .exceptionally(cause -> {
- // log the messages, since we DONT ack, pulsar consumer will re-deliver the messages.
- log.error("Failed to flush the state updates of message {}", msg, cause);
- return null;
- });
+ .thenRun(() -> processResult(msg, result, processAt, doneProcessing))
+ .exceptionally(cause -> {
+ // log the messages, since we DONT ack, pulsar consumer will re-deliver the messages.
+ log.error("Failed to flush the state updates of message {}", msg, cause);
+ return null;
+ });
} else {
processResult(msg, result, processAt, doneProcessing);
}
@@ -245,12 +245,12 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable {
log.info("Loading JAR files for function {} from jarFile {}", instanceConfig, jarFile);
// create the function class loader
fnCache.registerFunctionInstance(
- instanceConfig.getFunctionId(),
- instanceConfig.getInstanceId(),
- Arrays.asList(jarFile),
- Collections.emptyList());
+ instanceConfig.getFunctionId(),
+ instanceConfig.getInstanceId(),
+ Arrays.asList(jarFile),
+ Collections.emptyList());
log.info("Initialize function class loader for function {} at function cache manager",
- instanceConfig.getFunctionConfig().getName());
+ instanceConfig.getFunctionDetails().getName());
this.fnClassLoader = fnCache.getClassLoader(instanceConfig.getFunctionId());
if (null == fnClassLoader) {
@@ -267,46 +267,46 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable {
}
String tableNs = String.format(
- "%s_%s",
- instanceConfig.getFunctionConfig().getTenant(),
- instanceConfig.getFunctionConfig().getNamespace()
+ "%s_%s",
+ instanceConfig.getFunctionDetails().getTenant(),
+ instanceConfig.getFunctionDetails().getNamespace()
).replace('-', '_');
- String tableName = instanceConfig.getFunctionConfig().getName();
+ String tableName = instanceConfig.getFunctionDetails().getName();
// TODO (sijie): use endpoint for now
StorageClientSettings settings = StorageClientSettings.newBuilder()
- .addEndpoints(NetUtils.parseEndpoint(stateStorageServiceUrl))
- .clientName("function-" + tableNs + "/" + tableName)
- .build();
+ .addEndpoints(NetUtils.parseEndpoint(stateStorageServiceUrl))
+ .clientName("function-" + tableNs + "/" + tableName)
+ .build();
// TODO (sijie): provide a better way to provision the state table for functions
try (StorageAdminClient storageAdminClient = StorageClientBuilder.newBuilder()
- .withSettings(settings)
- .buildAdmin()) {
+ .withSettings(settings)
+ .buildAdmin()) {
try {
result(storageAdminClient.getStream(tableNs, tableName));
} catch (NamespaceNotFoundException nnfe) {
result(storageAdminClient.createNamespace(tableNs, NamespaceConfiguration.newBuilder()
- .setDefaultStreamConf(DEFAULT_STREAM_CONF)
- .build()));
+ .setDefaultStreamConf(DEFAULT_STREAM_CONF)
+ .build()));
result(storageAdminClient.createStream(tableNs, tableName, DEFAULT_STREAM_CONF));
} catch (StreamNotFoundException snfe) {
result(storageAdminClient.createStream(tableNs, tableName, DEFAULT_STREAM_CONF));
}
}
- log.info("Starting state table for function {}", instanceConfig.getFunctionConfig().getName());
+ log.info("Starting state table for function {}", instanceConfig.getFunctionDetails().getName());
this.storageClient = StorageClientBuilder.newBuilder()
- .withSettings(settings)
- .withNamespace(tableNs)
- .build();
+ .withSettings(settings)
+ .withNamespace(tableNs)
+ .build();
this.stateTable = result(storageClient.openTable(tableName));
}
private void processResult(InputMessage msg,
JavaExecutionResult result,
long startTime, long endTime) {
- if (result.getUserException() != null) {
+ if (result.getUserException() != null) {
log.info("Encountered user exception when processing message {}", msg, result.getUserException());
stats.incrementUserExceptions(result.getUserException());
processor.handleProcessException(msg, result.getUserException());
@@ -316,7 +316,7 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable {
processor.handleProcessException(msg, result.getSystemException());
} else {
stats.incrementSuccessfullyProcessed(endTime - startTime);
- if (result.getResult() != null && instanceConfig.getFunctionConfig().getOutput() != null) {
+ if (result.getResult() != null && instanceConfig.getFunctionDetails().getOutput() != null) {
byte[] output;
try {
output = outputSerDe.serialize(result.getResult());
@@ -340,9 +340,9 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable {
private void sendOutputMessage(InputMessage srcMsg,
byte[] output) {
MessageBuilder msgBuilder = MessageBuilder.create()
- .setContent(output)
- .setProperty("__pfn_input_topic__", srcMsg.getTopicName())
- .setProperty("__pfn_input_msg_id__", new String(Base64.getEncoder().encode(srcMsg.getActualMessage().getMessageId().toByteArray())));
+ .setContent(output)
+ .setProperty("__pfn_input_topic__", srcMsg.getTopicName())
+ .setProperty("__pfn_input_msg_id__", new String(Base64.getEncoder().encode(srcMsg.getActualMessage().getMessageId().toByteArray())));
processor.sendOutputMessage(srcMsg, msgBuilder);
}
@@ -367,8 +367,8 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable {
// once the thread quits, clean up the instance
fnCache.unregisterFunctionInstance(
- instanceConfig.getFunctionId(),
- instanceConfig.getInstanceId());
+ instanceConfig.getFunctionId(),
+ instanceConfig.getInstanceId());
log.info("Unloading JAR files for function {}", instanceConfig);
}
@@ -415,7 +415,7 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable {
private static void addSystemMetrics(String metricName, double value, InstanceCommunication.MetricsData.Builder bldr) {
InstanceCommunication.MetricsData.DataDigest digest =
InstanceCommunication.MetricsData.DataDigest.newBuilder()
- .setCount(value).setSum(value).setMax(value).setMin(0).build();
+ .setCount(value).setSum(value).setMax(value).setMin(0).build();
bldr.putMetrics(metricName, digest);
}
@@ -449,8 +449,8 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable {
private void setupSerDe(Class<?>[] typeArgs, ClassLoader clsLoader) {
this.inputSerDe = new HashMap<>();
- instanceConfig.getFunctionConfig().getCustomSerdeInputsMap().forEach((k, v) -> this.inputSerDe.put(k, initializeSerDe(v, clsLoader, typeArgs, true)));
- for (String topicName : instanceConfig.getFunctionConfig().getInputsList()) {
+ instanceConfig.getFunctionDetails().getCustomSerdeInputsMap().forEach((k, v) -> this.inputSerDe.put(k, initializeSerDe(v, clsLoader, typeArgs, true)));
+ for (String topicName : instanceConfig.getFunctionDetails().getInputsList()) {
this.inputSerDe.put(topicName, initializeDefaultSerDe(typeArgs, true));
}
@@ -473,12 +473,12 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable {
}
if (!Void.class.equals(typeArgs[1])) { // return type is not `Void.class`
- if (instanceConfig.getFunctionConfig().getOutputSerdeClassName() == null
- || instanceConfig.getFunctionConfig().getOutputSerdeClassName().isEmpty()
- || instanceConfig.getFunctionConfig().getOutputSerdeClassName().equals(DefaultSerDe.class.getName())) {
+ if (instanceConfig.getFunctionDetails().getOutputSerdeClassName() == null
+ || instanceConfig.getFunctionDetails().getOutputSerdeClassName().isEmpty()
+ || instanceConfig.getFunctionDetails().getOutputSerdeClassName().equals(DefaultSerDe.class.getName())) {
outputSerDe = initializeDefaultSerDe(typeArgs, false);
} else {
- this.outputSerDe = initializeSerDe(instanceConfig.getFunctionConfig().getOutputSerdeClassName(), clsLoader, typeArgs, false);
+ this.outputSerDe = initializeSerDe(instanceConfig.getFunctionDetails().getOutputSerdeClassName(), clsLoader, typeArgs, false);
}
Class<?>[] outputSerdeTypeArgs = TypeResolver.resolveRawArguments(SerDe.class, outputSerDe.getClass());
if (outputSerDe.getClass().getName().equals(DefaultSerDe.class.getName())) {
@@ -493,10 +493,10 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable {
}
private void setupLogHandler() {
- if (instanceConfig.getFunctionConfig().getLogTopic() != null &&
- !instanceConfig.getFunctionConfig().getLogTopic().isEmpty()) {
- logAppender = new LogAppender(client, instanceConfig.getFunctionConfig().getLogTopic(),
- FunctionConfigUtils.getFullyQualifiedName(instanceConfig.getFunctionConfig()));
+ if (instanceConfig.getFunctionDetails().getLogTopic() != null &&
+ !instanceConfig.getFunctionDetails().getLogTopic().isEmpty()) {
+ logAppender = new LogAppender(client, instanceConfig.getFunctionDetails().getLogTopic(),
+ FunctionDetailsUtils.getFullyQualifiedName(instanceConfig.getFunctionDetails()));
logAppender.start();
}
}
@@ -521,4 +521,4 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable {
}
config.getRootLogger().removeAppender(logAppender.getName());
}
-}
+}
\ No newline at end of file
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/AtLeastOnceProcessor.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/AtLeastOnceProcessor.java
index 740fd5c..38c1015 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/AtLeastOnceProcessor.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/AtLeastOnceProcessor.java
@@ -29,7 +29,7 @@ import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.functions.instance.InputMessage;
import org.apache.pulsar.functions.instance.producers.AbstractOneOuputTopicProducers;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails;
/**
* A message processor that process messages at-most-once.
@@ -41,10 +41,10 @@ public class AtLeastOnceProcessor extends MessageProcessorBase {
private Producer producer;
AtLeastOnceProcessor(PulsarClient client,
- FunctionConfig functionConfig,
+ FunctionDetails functionDetails,
SubscriptionType subType,
LinkedBlockingDeque<InputMessage> processQueue) {
- super(client, functionConfig, subType, processQueue);
+ super(client, functionDetails, subType, processQueue);
}
@Override
@@ -71,7 +71,7 @@ public class AtLeastOnceProcessor extends MessageProcessorBase {
try {
producer.close();
} catch (PulsarClientException e) {
- log.warn("Fail to close producer for processor {}", functionConfig.getOutput(), e);
+ log.warn("Fail to close producer for processor {}", functionDetails.getOutput(), e);
}
}
}
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/AtMostOnceProcessor.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/AtMostOnceProcessor.java
index 1713b73..d9f4eb1 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/AtMostOnceProcessor.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/AtMostOnceProcessor.java
@@ -28,7 +28,7 @@ import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.functions.instance.InputMessage;
import org.apache.pulsar.functions.instance.producers.AbstractOneOuputTopicProducers;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails;
/**
* A message processor that process messages at-most-once.
@@ -39,16 +39,16 @@ class AtMostOnceProcessor extends MessageProcessorBase {
private Producer producer;
AtMostOnceProcessor(PulsarClient client,
- FunctionConfig functionConfig,
+ FunctionDetails functionDetails,
SubscriptionType subType,
LinkedBlockingDeque<InputMessage> processQueue) {
- super(client, functionConfig, subType, processQueue);
+ super(client, functionDetails, subType, processQueue);
}
@Override
protected void postReceiveMessage(InputMessage message) {
super.postReceiveMessage(message);
- if (functionConfig.getAutoAck()) {
+ if (functionDetails.getAutoAck()) {
message.ack();
}
}
@@ -75,7 +75,7 @@ class AtMostOnceProcessor extends MessageProcessorBase {
try {
producer.close();
} catch (PulsarClientException e) {
- log.warn("Fail to close producer for processor {}", functionConfig.getOutput(), e);
+ log.warn("Fail to close producer for processor {}", functionDetails.getOutput(), e);
}
}
}
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/EffectivelyOnceProcessor.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/EffectivelyOnceProcessor.java
index f039cdf..5841026 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/EffectivelyOnceProcessor.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/EffectivelyOnceProcessor.java
@@ -38,8 +38,8 @@ import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.functions.instance.InputMessage;
import org.apache.pulsar.functions.instance.producers.MultiConsumersOneOuputTopicProducers;
import org.apache.pulsar.functions.instance.producers.Producers;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig;
-import org.apache.pulsar.functions.utils.FunctionConfigUtils;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails;
+import org.apache.pulsar.functions.utils.FunctionDetailsUtils;
import org.apache.pulsar.functions.utils.Utils;
/**
@@ -54,9 +54,9 @@ class EffectivelyOnceProcessor extends MessageProcessorBase implements ConsumerE
protected Producers outputProducer;
EffectivelyOnceProcessor(PulsarClient client,
- FunctionConfig functionConfig,
+ FunctionDetails functionDetails,
LinkedBlockingDeque<InputMessage> processQueue) {
- super(client, functionConfig, SubscriptionType.Failover, processQueue);
+ super(client, functionDetails, SubscriptionType.Failover, processQueue);
}
/**
@@ -180,7 +180,7 @@ class EffectivelyOnceProcessor extends MessageProcessorBase implements ConsumerE
.thenAccept(messageId -> inputMsg.ackCumulative())
.exceptionally(cause -> {
log.error("Failed to send the process result {} of message {} to output topic {}",
- outputMsg, inputMsg, functionConfig.getOutput(), cause);
+ outputMsg, inputMsg, functionDetails.getOutput(), cause);
handleProcessException(inputMsg.getTopicName());
return null;
});
@@ -251,7 +251,7 @@ class EffectivelyOnceProcessor extends MessageProcessorBase implements ConsumerE
srcTopic,
client.subscribe(
srcTopic,
- FunctionConfigUtils.getFullyQualifiedName(functionConfig),
+ FunctionDetailsUtils.getFullyQualifiedName(functionDetails),
conf
));
} catch (PulsarClientException e) {
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/MessageProcessor.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/MessageProcessor.java
index b475e9f..1167d73 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/MessageProcessor.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/MessageProcessor.java
@@ -27,8 +27,8 @@ import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.functions.api.SerDe;
import org.apache.pulsar.functions.instance.InputMessage;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig.ProcessingGuarantees;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails.ProcessingGuarantees;
/**
* A processor that processes messages, used by {@link org.apache.pulsar.functions.instance.JavaInstanceRunnable}.
@@ -37,12 +37,12 @@ import org.apache.pulsar.functions.proto.Function.FunctionConfig.ProcessingGuara
public interface MessageProcessor extends AutoCloseable {
static MessageProcessor create(PulsarClient client,
- FunctionConfig functionConfig,
+ FunctionDetails functionDetails,
LinkedBlockingDeque<InputMessage> processQeueue) {
- FunctionConfig.SubscriptionType fnSubType = functionConfig.getSubscriptionType();
- ProcessingGuarantees processingGuarantees = functionConfig.getProcessingGuarantees();
+ FunctionDetails.SubscriptionType fnSubType = functionDetails.getSubscriptionType();
+ ProcessingGuarantees processingGuarantees = functionDetails.getProcessingGuarantees();
SubscriptionType subType;
- if (null == fnSubType || FunctionConfig.SubscriptionType.SHARED == fnSubType) {
+ if (null == fnSubType || FunctionDetails.SubscriptionType.SHARED == fnSubType) {
subType = SubscriptionType.Shared;
} else {
subType = SubscriptionType.Failover;
@@ -51,18 +51,18 @@ public interface MessageProcessor extends AutoCloseable {
if (processingGuarantees == ProcessingGuarantees.EFFECTIVELY_ONCE) {
return new EffectivelyOnceProcessor(
client,
- functionConfig,
+ functionDetails,
processQeueue);
} else if (processingGuarantees == ProcessingGuarantees.ATMOST_ONCE) {
return new AtMostOnceProcessor(
client,
- functionConfig,
+ functionDetails,
subType,
processQeueue);
} else {
return new AtLeastOnceProcessor(
client,
- functionConfig,
+ functionDetails,
subType,
processQeueue);
}
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/MessageProcessorBase.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/MessageProcessorBase.java
index ddb5f79..46b2800 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/MessageProcessorBase.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/MessageProcessorBase.java
@@ -30,8 +30,8 @@ import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.functions.api.SerDe;
import org.apache.pulsar.functions.instance.InputMessage;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig;
-import org.apache.pulsar.functions.utils.FunctionConfigUtils;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails;
+import org.apache.pulsar.functions.utils.FunctionDetailsUtils;
/**
* The base implementation of {@link MessageProcessor}.
@@ -40,7 +40,7 @@ import org.apache.pulsar.functions.utils.FunctionConfigUtils;
abstract class MessageProcessorBase implements MessageProcessor {
protected final PulsarClient client;
- protected final FunctionConfig functionConfig;
+ protected final FunctionDetails functionDetails;
protected final SubscriptionType subType;
protected final LinkedBlockingDeque<InputMessage> processQueue;
@@ -51,11 +51,11 @@ abstract class MessageProcessorBase implements MessageProcessor {
protected SerDe outputSerDe;
protected MessageProcessorBase(PulsarClient client,
- FunctionConfig functionConfig,
+ FunctionDetails functionDetails,
SubscriptionType subType,
LinkedBlockingDeque<InputMessage> processQueue) {
this.client = client;
- this.functionConfig = functionConfig;
+ this.functionDetails = functionDetails;
this.subType = subType;
this.processQueue = processQueue;
this.inputConsumers = Maps.newConcurrentMap();
@@ -69,15 +69,15 @@ abstract class MessageProcessorBase implements MessageProcessor {
public void setupInput(Map<String, SerDe> inputSerDe) throws Exception {
log.info("Setting up input with input serdes: {}", inputSerDe);
this.inputSerDe = inputSerDe;
- for (Map.Entry<String, String> entry : functionConfig.getCustomSerdeInputsMap().entrySet()) {
+ for (Map.Entry<String, String> entry : functionDetails.getCustomSerdeInputsMap().entrySet()) {
ConsumerConfiguration conf = createConsumerConfiguration(entry.getKey());
this.inputConsumers.put(entry.getKey(), client.subscribe(entry.getKey(),
- FunctionConfigUtils.getFullyQualifiedName(functionConfig), conf));
+ FunctionDetailsUtils.getFullyQualifiedName(functionDetails), conf));
}
- for (String topicName : functionConfig.getInputsList()) {
+ for (String topicName : functionDetails.getInputsList()) {
ConsumerConfiguration conf = createConsumerConfiguration(topicName);
this.inputConsumers.put(topicName, client.subscribe(topicName,
- FunctionConfigUtils.getFullyQualifiedName(functionConfig), conf));
+ FunctionDetailsUtils.getFullyQualifiedName(functionDetails), conf));
}
}
@@ -126,9 +126,9 @@ abstract class MessageProcessorBase implements MessageProcessor {
public void setupOutput(SerDe outputSerDe) throws Exception {
this.outputSerDe = outputSerDe;
- String outputTopic = functionConfig.getOutput();
+ String outputTopic = functionDetails.getOutput();
if (outputTopic != null
- && !functionConfig.getOutput().isEmpty()
+ && !functionDetails.getOutput().isEmpty()
&& outputSerDe != null) {
log.info("Starting producer for output topic {}", outputTopic);
initializeOutputProducer(outputTopic);
diff --git a/pulsar-functions/instance/src/main/python/Function_pb2.py b/pulsar-functions/instance/src/main/python/Function_pb2.py
index 95fcd7a..eeb1fab 100644
--- a/pulsar-functions/instance/src/main/python/Function_pb2.py
+++ b/pulsar-functions/instance/src/main/python/Function_pb2.py
@@ -16,8 +16,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-
+#
+
# -*- encoding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
@@ -38,17 +38,17 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
- name='pulsar-functions/proto/src/main/proto/Function.proto',
+ name='Function.proto',
package='proto',
syntax='proto3',
- serialized_pb=_b('\n4pulsar-functions/proto/src/main/proto/Function.proto\x12\x05proto\"\x98\x06\n\x0e\x46unctionConfig\x12\x0e\n\x06tenant\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x11\n\tclassName\x18\x04 \x01(\t\x12G\n\x11\x63ustomSerdeInputs\x18\x05 \x03(\x0b\x32,.proto.FunctionConfig.CustomSerdeInputsEntry\x12\x1c\n\x14outputSerdeClassName\x18\x06 \x01(\t\x12\x0e\n\x06output\x18\x07 \x01(\t\x12\x10\n\x08logTopic\x18\x08 \x01(\t\x12H [...]
+ serialized_pb=_b('\n\x0e\x46unction.proto\x12\x05proto\"\xac\x06\n\x0f\x46unctionDetails\x12\x0e\n\x06tenant\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x11\n\tclassName\x18\x04 \x01(\t\x12H\n\x11\x63ustomSerdeInputs\x18\x05 \x03(\x0b\x32-.proto.FunctionDetails.CustomSerdeInputsEntry\x12\x1c\n\x14outputSerdeClassName\x18\x06 \x01(\t\x12\x0e\n\x06output\x18\x07 \x01(\t\x12\x10\n\x08logTopic\x18\x08 \x01(\t\x12I\n\x14processingGuarantees\x18 [...]
)
-_FUNCTIONCONFIG_PROCESSINGGUARANTEES = _descriptor.EnumDescriptor(
+_FUNCTIONDETAILS_PROCESSINGGUARANTEES = _descriptor.EnumDescriptor(
name='ProcessingGuarantees',
- full_name='proto.FunctionConfig.ProcessingGuarantees',
+ full_name='proto.FunctionDetails.ProcessingGuarantees',
filename=None,
file=DESCRIPTOR,
values=[
@@ -67,14 +67,14 @@ _FUNCTIONCONFIG_PROCESSINGGUARANTEES = _descriptor.EnumDescriptor(
],
containing_type=None,
options=None,
- serialized_start=697,
- serialized_end=776,
+ serialized_start=679,
+ serialized_end=758,
)
-_sym_db.RegisterEnumDescriptor(_FUNCTIONCONFIG_PROCESSINGGUARANTEES)
+_sym_db.RegisterEnumDescriptor(_FUNCTIONDETAILS_PROCESSINGGUARANTEES)
-_FUNCTIONCONFIG_SUBSCRIPTIONTYPE = _descriptor.EnumDescriptor(
+_FUNCTIONDETAILS_SUBSCRIPTIONTYPE = _descriptor.EnumDescriptor(
name='SubscriptionType',
- full_name='proto.FunctionConfig.SubscriptionType',
+ full_name='proto.FunctionDetails.SubscriptionType',
filename=None,
file=DESCRIPTOR,
values=[
@@ -89,14 +89,14 @@ _FUNCTIONCONFIG_SUBSCRIPTIONTYPE = _descriptor.EnumDescriptor(
],
containing_type=None,
options=None,
- serialized_start=778,
- serialized_end=823,
+ serialized_start=760,
+ serialized_end=805,
)
-_sym_db.RegisterEnumDescriptor(_FUNCTIONCONFIG_SUBSCRIPTIONTYPE)
+_sym_db.RegisterEnumDescriptor(_FUNCTIONDETAILS_SUBSCRIPTIONTYPE)
-_FUNCTIONCONFIG_RUNTIME = _descriptor.EnumDescriptor(
+_FUNCTIONDETAILS_RUNTIME = _descriptor.EnumDescriptor(
name='Runtime',
- full_name='proto.FunctionConfig.Runtime',
+ full_name='proto.FunctionDetails.Runtime',
filename=None,
file=DESCRIPTOR,
values=[
@@ -111,28 +111,28 @@ _FUNCTIONCONFIG_RUNTIME = _descriptor.EnumDescriptor(
],
containing_type=None,
options=None,
- serialized_start=825,
- serialized_end=856,
+ serialized_start=807,
+ serialized_end=838,
)
-_sym_db.RegisterEnumDescriptor(_FUNCTIONCONFIG_RUNTIME)
+_sym_db.RegisterEnumDescriptor(_FUNCTIONDETAILS_RUNTIME)
-_FUNCTIONCONFIG_CUSTOMSERDEINPUTSENTRY = _descriptor.Descriptor(
+_FUNCTIONDETAILS_CUSTOMSERDEINPUTSENTRY = _descriptor.Descriptor(
name='CustomSerdeInputsEntry',
- full_name='proto.FunctionConfig.CustomSerdeInputsEntry',
+ full_name='proto.FunctionDetails.CustomSerdeInputsEntry',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='key', full_name='proto.FunctionConfig.CustomSerdeInputsEntry.key', index=0,
+ name='key', full_name='proto.FunctionDetails.CustomSerdeInputsEntry.key', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
- name='value', full_name='proto.FunctionConfig.CustomSerdeInputsEntry.value', index=1,
+ name='value', full_name='proto.FunctionDetails.CustomSerdeInputsEntry.value', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
@@ -150,26 +150,26 @@ _FUNCTIONCONFIG_CUSTOMSERDEINPUTSENTRY = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=588,
- serialized_end=644,
+ serialized_start=570,
+ serialized_end=626,
)
-_FUNCTIONCONFIG_USERCONFIGENTRY = _descriptor.Descriptor(
+_FUNCTIONDETAILS_USERCONFIGENTRY = _descriptor.Descriptor(
name='UserConfigEntry',
- full_name='proto.FunctionConfig.UserConfigEntry',
+ full_name='proto.FunctionDetails.UserConfigEntry',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='key', full_name='proto.FunctionConfig.UserConfigEntry.key', index=0,
+ name='key', full_name='proto.FunctionDetails.UserConfigEntry.key', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
- name='value', full_name='proto.FunctionConfig.UserConfigEntry.value', index=1,
+ name='value', full_name='proto.FunctionDetails.UserConfigEntry.value', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
@@ -187,130 +187,137 @@ _FUNCTIONCONFIG_USERCONFIGENTRY = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=646,
- serialized_end=695,
+ serialized_start=628,
+ serialized_end=677,
)
-_FUNCTIONCONFIG = _descriptor.Descriptor(
- name='FunctionConfig',
- full_name='proto.FunctionConfig',
+_FUNCTIONDETAILS = _descriptor.Descriptor(
+ name='FunctionDetails',
+ full_name='proto.FunctionDetails',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='tenant', full_name='proto.FunctionConfig.tenant', index=0,
+ name='tenant', full_name='proto.FunctionDetails.tenant', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
- name='namespace', full_name='proto.FunctionConfig.namespace', index=1,
+ name='namespace', full_name='proto.FunctionDetails.namespace', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
- name='name', full_name='proto.FunctionConfig.name', index=2,
+ name='name', full_name='proto.FunctionDetails.name', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
- name='className', full_name='proto.FunctionConfig.className', index=3,
+ name='className', full_name='proto.FunctionDetails.className', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
- name='customSerdeInputs', full_name='proto.FunctionConfig.customSerdeInputs', index=4,
+ name='customSerdeInputs', full_name='proto.FunctionDetails.customSerdeInputs', index=4,
number=5, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
- name='outputSerdeClassName', full_name='proto.FunctionConfig.outputSerdeClassName', index=5,
+ name='outputSerdeClassName', full_name='proto.FunctionDetails.outputSerdeClassName', index=5,
number=6, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
- name='output', full_name='proto.FunctionConfig.output', index=6,
+ name='output', full_name='proto.FunctionDetails.output', index=6,
number=7, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
- name='logTopic', full_name='proto.FunctionConfig.logTopic', index=7,
+ name='logTopic', full_name='proto.FunctionDetails.logTopic', index=7,
number=8, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
- name='processingGuarantees', full_name='proto.FunctionConfig.processingGuarantees', index=8,
+ name='processingGuarantees', full_name='proto.FunctionDetails.processingGuarantees', index=8,
number=9, type=14, cpp_type=8, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
- name='userConfig', full_name='proto.FunctionConfig.userConfig', index=9,
+ name='userConfig', full_name='proto.FunctionDetails.userConfig', index=9,
number=10, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
- name='subscriptionType', full_name='proto.FunctionConfig.subscriptionType', index=10,
+ name='subscriptionType', full_name='proto.FunctionDetails.subscriptionType', index=10,
number=11, type=14, cpp_type=8, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
- name='runtime', full_name='proto.FunctionConfig.runtime', index=11,
+ name='runtime', full_name='proto.FunctionDetails.runtime', index=11,
number=12, type=14, cpp_type=8, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
- name='autoAck', full_name='proto.FunctionConfig.autoAck', index=12,
+ name='autoAck', full_name='proto.FunctionDetails.autoAck', index=12,
number=13, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
- name='inputs', full_name='proto.FunctionConfig.inputs', index=13,
+ name='inputs', full_name='proto.FunctionDetails.inputs', index=13,
number=14, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
- name='parallelism', full_name='proto.FunctionConfig.parallelism', index=14,
+ name='parallelism', full_name='proto.FunctionDetails.parallelism', index=14,
number=15, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
+ _descriptor.FieldDescriptor(
+ name='fqfn', full_name='proto.FunctionDetails.fqfn', index=15,
+ number=16, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=_b("").decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None, file=DESCRIPTOR),
],
extensions=[
],
- nested_types=[_FUNCTIONCONFIG_CUSTOMSERDEINPUTSENTRY, _FUNCTIONCONFIG_USERCONFIGENTRY, ],
+ nested_types=[_FUNCTIONDETAILS_CUSTOMSERDEINPUTSENTRY, _FUNCTIONDETAILS_USERCONFIGENTRY, ],
enum_types=[
- _FUNCTIONCONFIG_PROCESSINGGUARANTEES,
- _FUNCTIONCONFIG_SUBSCRIPTIONTYPE,
- _FUNCTIONCONFIG_RUNTIME,
+ _FUNCTIONDETAILS_PROCESSINGGUARANTEES,
+ _FUNCTIONDETAILS_SUBSCRIPTIONTYPE,
+ _FUNCTIONDETAILS_RUNTIME,
],
options=None,
is_extendable=False,
@@ -318,8 +325,8 @@ _FUNCTIONCONFIG = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=64,
- serialized_end=856,
+ serialized_start=26,
+ serialized_end=838,
)
@@ -349,8 +356,8 @@ _PACKAGELOCATIONMETADATA = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=858,
- serialized_end=904,
+ serialized_start=840,
+ serialized_end=886,
)
@@ -362,7 +369,7 @@ _FUNCTIONMETADATA = _descriptor.Descriptor(
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
- name='functionConfig', full_name='proto.FunctionMetaData.functionConfig', index=0,
+ name='functionDetails', full_name='proto.FunctionMetaData.functionDetails', index=0,
number=1, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
@@ -401,46 +408,8 @@ _FUNCTIONMETADATA = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=907,
- serialized_end=1066,
-)
-
-
-_SNAPSHOT = _descriptor.Descriptor(
- name='Snapshot',
- full_name='proto.Snapshot',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='functionMetaDataList', full_name='proto.Snapshot.functionMetaDataList', index=0,
- number=1, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='lastAppliedMessageId', full_name='proto.Snapshot.lastAppliedMessageId', index=1,
- number=2, type=12, cpp_type=9, label=1,
- has_default_value=False, default_value=_b(""),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- options=None, file=DESCRIPTOR),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=1068,
- serialized_end=1163,
+ serialized_start=889,
+ serialized_end=1050,
)
@@ -477,8 +446,8 @@ _INSTANCE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=1165,
- serialized_end=1246,
+ serialized_start=1052,
+ serialized_end=1133,
)
@@ -515,87 +484,78 @@ _ASSIGNMENT = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=1248,
- serialized_end=1313,
+ serialized_start=1135,
+ serialized_end=1200,
)
-_FUNCTIONCONFIG_CUSTOMSERDEINPUTSENTRY.containing_type = _FUNCTIONCONFIG
-_FUNCTIONCONFIG_USERCONFIGENTRY.containing_type = _FUNCTIONCONFIG
-_FUNCTIONCONFIG.fields_by_name['customSerdeInputs'].message_type = _FUNCTIONCONFIG_CUSTOMSERDEINPUTSENTRY
-_FUNCTIONCONFIG.fields_by_name['processingGuarantees'].enum_type = _FUNCTIONCONFIG_PROCESSINGGUARANTEES
-_FUNCTIONCONFIG.fields_by_name['userConfig'].message_type = _FUNCTIONCONFIG_USERCONFIGENTRY
-_FUNCTIONCONFIG.fields_by_name['subscriptionType'].enum_type = _FUNCTIONCONFIG_SUBSCRIPTIONTYPE
-_FUNCTIONCONFIG.fields_by_name['runtime'].enum_type = _FUNCTIONCONFIG_RUNTIME
-_FUNCTIONCONFIG_PROCESSINGGUARANTEES.containing_type = _FUNCTIONCONFIG
-_FUNCTIONCONFIG_SUBSCRIPTIONTYPE.containing_type = _FUNCTIONCONFIG
-_FUNCTIONCONFIG_RUNTIME.containing_type = _FUNCTIONCONFIG
-_FUNCTIONMETADATA.fields_by_name['functionConfig'].message_type = _FUNCTIONCONFIG
+_FUNCTIONDETAILS_CUSTOMSERDEINPUTSENTRY.containing_type = _FUNCTIONDETAILS
+_FUNCTIONDETAILS_USERCONFIGENTRY.containing_type = _FUNCTIONDETAILS
+_FUNCTIONDETAILS.fields_by_name['customSerdeInputs'].message_type = _FUNCTIONDETAILS_CUSTOMSERDEINPUTSENTRY
+_FUNCTIONDETAILS.fields_by_name['processingGuarantees'].enum_type = _FUNCTIONDETAILS_PROCESSINGGUARANTEES
+_FUNCTIONDETAILS.fields_by_name['userConfig'].message_type = _FUNCTIONDETAILS_USERCONFIGENTRY
+_FUNCTIONDETAILS.fields_by_name['subscriptionType'].enum_type = _FUNCTIONDETAILS_SUBSCRIPTIONTYPE
+_FUNCTIONDETAILS.fields_by_name['runtime'].enum_type = _FUNCTIONDETAILS_RUNTIME
+_FUNCTIONDETAILS_PROCESSINGGUARANTEES.containing_type = _FUNCTIONDETAILS
+_FUNCTIONDETAILS_SUBSCRIPTIONTYPE.containing_type = _FUNCTIONDETAILS
+_FUNCTIONDETAILS_RUNTIME.containing_type = _FUNCTIONDETAILS
+_FUNCTIONMETADATA.fields_by_name['functionDetails'].message_type = _FUNCTIONDETAILS
_FUNCTIONMETADATA.fields_by_name['packageLocation'].message_type = _PACKAGELOCATIONMETADATA
-_SNAPSHOT.fields_by_name['functionMetaDataList'].message_type = _FUNCTIONMETADATA
_INSTANCE.fields_by_name['functionMetaData'].message_type = _FUNCTIONMETADATA
_ASSIGNMENT.fields_by_name['instance'].message_type = _INSTANCE
-DESCRIPTOR.message_types_by_name['FunctionConfig'] = _FUNCTIONCONFIG
+DESCRIPTOR.message_types_by_name['FunctionDetails'] = _FUNCTIONDETAILS
DESCRIPTOR.message_types_by_name['PackageLocationMetaData'] = _PACKAGELOCATIONMETADATA
DESCRIPTOR.message_types_by_name['FunctionMetaData'] = _FUNCTIONMETADATA
-DESCRIPTOR.message_types_by_name['Snapshot'] = _SNAPSHOT
DESCRIPTOR.message_types_by_name['Instance'] = _INSTANCE
DESCRIPTOR.message_types_by_name['Assignment'] = _ASSIGNMENT
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
-FunctionConfig = _reflection.GeneratedProtocolMessageType('FunctionConfig', (_message.Message,), dict(
+FunctionDetails = _reflection.GeneratedProtocolMessageType('FunctionDetails', (_message.Message,), dict(
CustomSerdeInputsEntry = _reflection.GeneratedProtocolMessageType('CustomSerdeInputsEntry', (_message.Message,), dict(
- DESCRIPTOR = _FUNCTIONCONFIG_CUSTOMSERDEINPUTSENTRY,
- __module__ = 'pulsar_functions.proto.src.main.proto.Function_pb2'
- # @@protoc_insertion_point(class_scope:proto.FunctionConfig.CustomSerdeInputsEntry)
+ DESCRIPTOR = _FUNCTIONDETAILS_CUSTOMSERDEINPUTSENTRY,
+ __module__ = 'Function_pb2'
+ # @@protoc_insertion_point(class_scope:proto.FunctionDetails.CustomSerdeInputsEntry)
))
,
UserConfigEntry = _reflection.GeneratedProtocolMessageType('UserConfigEntry', (_message.Message,), dict(
- DESCRIPTOR = _FUNCTIONCONFIG_USERCONFIGENTRY,
- __module__ = 'pulsar_functions.proto.src.main.proto.Function_pb2'
- # @@protoc_insertion_point(class_scope:proto.FunctionConfig.UserConfigEntry)
+ DESCRIPTOR = _FUNCTIONDETAILS_USERCONFIGENTRY,
+ __module__ = 'Function_pb2'
+ # @@protoc_insertion_point(class_scope:proto.FunctionDetails.UserConfigEntry)
))
,
- DESCRIPTOR = _FUNCTIONCONFIG,
- __module__ = 'pulsar_functions.proto.src.main.proto.Function_pb2'
- # @@protoc_insertion_point(class_scope:proto.FunctionConfig)
+ DESCRIPTOR = _FUNCTIONDETAILS,
+ __module__ = 'Function_pb2'
+ # @@protoc_insertion_point(class_scope:proto.FunctionDetails)
))
-_sym_db.RegisterMessage(FunctionConfig)
-_sym_db.RegisterMessage(FunctionConfig.CustomSerdeInputsEntry)
-_sym_db.RegisterMessage(FunctionConfig.UserConfigEntry)
+_sym_db.RegisterMessage(FunctionDetails)
+_sym_db.RegisterMessage(FunctionDetails.CustomSerdeInputsEntry)
+_sym_db.RegisterMessage(FunctionDetails.UserConfigEntry)
PackageLocationMetaData = _reflection.GeneratedProtocolMessageType('PackageLocationMetaData', (_message.Message,), dict(
DESCRIPTOR = _PACKAGELOCATIONMETADATA,
- __module__ = 'pulsar_functions.proto.src.main.proto.Function_pb2'
+ __module__ = 'Function_pb2'
# @@protoc_insertion_point(class_scope:proto.PackageLocationMetaData)
))
_sym_db.RegisterMessage(PackageLocationMetaData)
FunctionMetaData = _reflection.GeneratedProtocolMessageType('FunctionMetaData', (_message.Message,), dict(
DESCRIPTOR = _FUNCTIONMETADATA,
- __module__ = 'pulsar_functions.proto.src.main.proto.Function_pb2'
+ __module__ = 'Function_pb2'
# @@protoc_insertion_point(class_scope:proto.FunctionMetaData)
))
_sym_db.RegisterMessage(FunctionMetaData)
-Snapshot = _reflection.GeneratedProtocolMessageType('Snapshot', (_message.Message,), dict(
- DESCRIPTOR = _SNAPSHOT,
- __module__ = 'pulsar_functions.proto.src.main.proto.Function_pb2'
- # @@protoc_insertion_point(class_scope:proto.Snapshot)
- ))
-_sym_db.RegisterMessage(Snapshot)
-
Instance = _reflection.GeneratedProtocolMessageType('Instance', (_message.Message,), dict(
DESCRIPTOR = _INSTANCE,
- __module__ = 'pulsar_functions.proto.src.main.proto.Function_pb2'
+ __module__ = 'Function_pb2'
# @@protoc_insertion_point(class_scope:proto.Instance)
))
_sym_db.RegisterMessage(Instance)
Assignment = _reflection.GeneratedProtocolMessageType('Assignment', (_message.Message,), dict(
DESCRIPTOR = _ASSIGNMENT,
- __module__ = 'pulsar_functions.proto.src.main.proto.Function_pb2'
+ __module__ = 'Function_pb2'
# @@protoc_insertion_point(class_scope:proto.Assignment)
))
_sym_db.RegisterMessage(Assignment)
@@ -603,8 +563,8 @@ _sym_db.RegisterMessage(Assignment)
DESCRIPTOR.has_options = True
DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n!org.apache.pulsar.functions.protoB\010Function'))
-_FUNCTIONCONFIG_CUSTOMSERDEINPUTSENTRY.has_options = True
-_FUNCTIONCONFIG_CUSTOMSERDEINPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_FUNCTIONCONFIG_USERCONFIGENTRY.has_options = True
-_FUNCTIONCONFIG_USERCONFIGENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
+_FUNCTIONDETAILS_CUSTOMSERDEINPUTSENTRY.has_options = True
+_FUNCTIONDETAILS_CUSTOMSERDEINPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
+_FUNCTIONDETAILS_USERCONFIGENTRY.has_options = True
+_FUNCTIONDETAILS_USERCONFIGENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
# @@protoc_insertion_point(module_scope)
diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py
index cc58526..c0165b6 100644
--- a/pulsar-functions/instance/src/main/python/python_instance.py
+++ b/pulsar-functions/instance/src/main/python/python_instance.py
@@ -40,7 +40,7 @@ import InstanceCommunication_pb2
Log = log.Log
# Equivalent of the InstanceConfig in Java
-InstanceConfig = namedtuple('InstanceConfig', 'instance_id function_id function_version function_config max_buffered_tuples')
+InstanceConfig = namedtuple('InstanceConfig', 'instance_id function_id function_version function_details max_buffered_tuples')
# This is the message that the consumers put on the queue for the function thread to process
InternalMessage = namedtuple('InternalMessage', 'message topic serde consumer')
InternalQuitMessage = namedtuple('InternalQuitMessage', 'quit')
@@ -95,8 +95,8 @@ class Stats(object):
return self.latency / self.nsuccessfullyprocessed
class PythonInstance(object):
- def __init__(self, instance_id, function_id, function_version, function_config, max_buffered_tuples, user_code, log_topic, pulsar_client):
- self.instance_config = InstanceConfig(instance_id, function_id, function_version, function_config, max_buffered_tuples)
+ def __init__(self, instance_id, function_id, function_version, function_details, max_buffered_tuples, user_code, log_topic, pulsar_client):
+ self.instance_config = InstanceConfig(instance_id, function_id, function_version, function_details, max_buffered_tuples)
self.user_code = user_code
self.queue = Queue.Queue(max_buffered_tuples)
self.log_topic_handler = None
@@ -110,9 +110,9 @@ class PythonInstance(object):
self.function_purefunction = None
self.producer = None
self.exeuction_thread = None
- self.atmost_once = self.instance_config.function_config.processingGuarantees == Function_pb2.FunctionConfig.ProcessingGuarantees.Value('ATMOST_ONCE')
- self.atleast_once = self.instance_config.function_config.processingGuarantees == Function_pb2.FunctionConfig.ProcessingGuarantees.Value('ATLEAST_ONCE')
- self.auto_ack = self.instance_config.function_config.autoAck
+ self.atmost_once = self.instance_config.function_details.processingGuarantees == Function_pb2.FunctionDetails.ProcessingGuarantees.Value('ATMOST_ONCE')
+ self.atleast_once = self.instance_config.function_details.processingGuarantees == Function_pb2.FunctionDetails.ProcessingGuarantees.Value('ATLEAST_ONCE')
+ self.auto_ack = self.instance_config.function_details.autoAck
self.contextimpl = None
self.total_stats = Stats()
self.current_stats = Stats()
@@ -122,10 +122,10 @@ class PythonInstance(object):
mode = pulsar._pulsar.ConsumerType.Exclusive
if self.atmost_once:
mode = pulsar._pulsar.ConsumerType.Shared
- subscription_name = str(self.instance_config.function_config.tenant) + "/" + \
- str(self.instance_config.function_config.namespace) + "/" + \
- str(self.instance_config.function_config.name)
- for topic, serde in self.instance_config.function_config.customSerdeInputs.items():
+ subscription_name = str(self.instance_config.function_details.tenant) + "/" + \
+ str(self.instance_config.function_details.namespace) + "/" + \
+ str(self.instance_config.function_details.name)
+ for topic, serde in self.instance_config.function_details.customSerdeInputs.items():
serde_kclass = util.import_class(os.path.dirname(self.user_code), serde)
self.input_serdes[topic] = serde_kclass()
Log.info("Setting up consumer for topic %s with subname %s" % (topic, subscription_name))
@@ -135,7 +135,7 @@ class PythonInstance(object):
message_listener=partial(self.message_listener, topic, self.input_serdes[topic])
)
- for topic in self.instance_config.function_config.inputs:
+ for topic in self.instance_config.function_details.inputs:
global DEFAULT_SERIALIZER
serde_kclass = util.import_class(os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
self.input_serdes[topic] = serde_kclass()
@@ -146,10 +146,10 @@ class PythonInstance(object):
message_listener=partial(self.message_listener, topic, self.input_serdes[topic])
)
- function_kclass = util.import_class(os.path.dirname(self.user_code), self.instance_config.function_config.className)
+ function_kclass = util.import_class(os.path.dirname(self.user_code), self.instance_config.function_details.className)
if function_kclass is None:
- Log.critical("Could not import User Function Module %s" % self.instance_config.function_config.className)
- raise NameError("Could not import User Function Module %s" % self.instance_config.function_config.className)
+ Log.critical("Could not import User Function Module %s" % self.instance_config.function_details.className)
+ raise NameError("Could not import User Function Module %s" % self.instance_config.function_details.className)
try:
self.function_class = function_kclass()
except:
@@ -233,9 +233,9 @@ class PythonInstance(object):
msg.consumer.acknowledge(msg.message)
def setup_output_serde(self):
- if self.instance_config.function_config.outputSerdeClassName != None and \
- len(self.instance_config.function_config.outputSerdeClassName) > 0:
- serde_kclass = util.import_class(os.path.dirname(self.user_code), self.instance_config.function_config.outputSerdeClassName)
+ if self.instance_config.function_details.outputSerdeClassName != None and \
+ len(self.instance_config.function_details.outputSerdeClassName) > 0:
+ serde_kclass = util.import_class(os.path.dirname(self.user_code), self.instance_config.function_details.outputSerdeClassName)
self.output_serde = serde_kclass()
else:
global DEFAULT_SERIALIZER
@@ -243,11 +243,11 @@ class PythonInstance(object):
self.output_serde = serde_kclass()
def setup_producer(self):
- if self.instance_config.function_config.output != None and \
- len(self.instance_config.function_config.output) > 0:
- Log.info("Setting up producer for topic %s" % self.instance_config.function_config.output)
+ if self.instance_config.function_details.output != None and \
+ len(self.instance_config.function_details.output) > 0:
+ Log.info("Setting up producer for topic %s" % self.instance_config.function_details.output)
self.producer = self.pulsar_client.create_producer(
- str(self.instance_config.function_config.output),
+ str(self.instance_config.function_details.output),
block_if_queue_full=True,
batching_enabled=True,
batching_max_publish_delay_ms=1,
diff --git a/pulsar-functions/instance/src/main/python/python_instance_main.py b/pulsar-functions/instance/src/main/python/python_instance_main.py
index 2638fa5..0e9d857 100644
--- a/pulsar-functions/instance/src/main/python/python_instance_main.py
+++ b/pulsar-functions/instance/src/main/python/python_instance_main.py
@@ -85,11 +85,11 @@ def main():
Log.info("Starting Python instance with %s" % str(args))
- function_config = Function_pb2.FunctionConfig()
- function_config.tenant = args.tenant
- function_config.namespace = args.namespace
- function_config.name = args.name
- function_config.className = args.function_classname
+ function_details = Function_pb2.FunctionDetails()
+ function_details.tenant = args.tenant
+ function_details.namespace = args.namespace
+ function_details.name = args.name
+ function_details.className = args.function_classname
if args.custom_serde_input_topics is None and args.input_topics is None:
Log.critical("Atleast one input topic must be present")
sys.exit(1)
@@ -100,27 +100,27 @@ def main():
Log.critical("CustomSerde InputTopcis and Serde classnames should match")
sys.exit(1)
for i in xrange(len(input_topics)):
- function_config.customSerdeInputs[input_topics[i]] = input_serde[i]
+ function_details.customSerdeInputs[input_topics[i]] = input_serde[i]
if args.input_topics is not None:
for topic in args.input_topics.split(","):
- function_config.inputs.append(topic)
+ function_details.inputs.append(topic)
if args.output_topic != None and len(args.output_topic) != 0:
- function_config.output = args.output_topic
+ function_details.output = args.output_topic
if args.output_serde_classname != None and len(args.output_serde_classname) != 0:
- function_config.outputSerdeClassName = args.output_serde_classname
- function_config.processingGuarantees = Function_pb2.FunctionConfig.ProcessingGuarantees.Value(args.processing_guarantees)
+ function_details.outputSerdeClassName = args.output_serde_classname
+ function_details.processingGuarantees = Function_pb2.FunctionDetails.ProcessingGuarantees.Value(args.processing_guarantees)
if args.auto_ack == "true":
- function_config.autoAck = True
+ function_details.autoAck = True
else:
- function_config.autoAck = False
+ function_details.autoAck = False
if args.user_config != None and len(args.user_config) != 0:
user_config = json.loads(args.user_config)
for (key, value) in user_config.items():
- function_config.userConfig[str(key)] = str(value)
+ function_details.userConfig[str(key)] = str(value)
pulsar_client = pulsar.Client(args.pulsar_serviceurl)
pyinstance = python_instance.PythonInstance(str(args.instance_id), str(args.function_id),
- str(args.function_version), function_config,
+ str(args.function_version), function_details,
int(args.max_buffered_tuples), str(args.py),
args.log_topic, pulsar_client)
pyinstance.run()
diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableProcessTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableProcessTest.java
index 8e099a6..1f76bff 100644
--- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableProcessTest.java
+++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableProcessTest.java
@@ -78,11 +78,11 @@ import org.apache.pulsar.functions.api.Function;
import org.apache.pulsar.functions.api.utils.DefaultSerDe;
import org.apache.pulsar.functions.instance.processors.AtLeastOnceProcessor;
import org.apache.pulsar.functions.instance.processors.MessageProcessor;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig.ProcessingGuarantees;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails.ProcessingGuarantees;
import org.apache.pulsar.functions.utils.Reflections;
import org.apache.pulsar.functions.utils.functioncache.FunctionCacheManager;
-import org.apache.pulsar.functions.utils.FunctionConfigUtils;
+import org.apache.pulsar.functions.utils.FunctionDetailsUtils;
import org.apache.pulsar.functions.utils.Utils;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
@@ -208,16 +208,16 @@ public class JavaInstanceRunnableProcessTest {
private static final String TEST_STORAGE_SERVICE_URL = "127.0.0.1:4181";
private static final LinkedBlockingQueue<String> voidFunctionQueue = new LinkedBlockingQueue<>();
- private FunctionConfig fnConfig;
+ private FunctionDetails functionDetails;
private InstanceConfig config;
private FunctionCacheManager fnCache;
private PulsarClient mockClient;
private FunctionStats mockFunctionStats;
private final Map<Pair<String, String>, ProducerInstance> mockProducers
- = Collections.synchronizedMap(new HashMap<>());
+ = Collections.synchronizedMap(new HashMap<>());
private final Map<Pair<String, String>, ConsumerInstance> mockConsumers
- = Collections.synchronizedMap(new HashMap<>());
+ = Collections.synchronizedMap(new HashMap<>());
private StorageClient mockStorageClient;
private Table<ByteBuf, ByteBuf> mockTable;
@@ -226,23 +226,23 @@ public class JavaInstanceRunnableProcessTest {
mockProducers.clear();
mockConsumers.clear();
- fnConfig = FunctionConfig.newBuilder()
- .setAutoAck(true)
- .setClassName(TestFunction.class.getName())
- .addInputs("test-src-topic")
- .setName("test-function")
- .setOutput("test-output-topic")
- .setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE)
- .setTenant("test-tenant")
- .setNamespace("test-namespace")
- .build();
+ functionDetails = FunctionDetails.newBuilder()
+ .setAutoAck(true)
+ .setClassName(TestFunction.class.getName())
+ .addInputs("test-src-topic")
+ .setName("test-function")
+ .setOutput("test-output-topic")
+ .setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE)
+ .setTenant("test-tenant")
+ .setNamespace("test-namespace")
+ .build();
config = new InstanceConfig();
config.setFunctionId("test-function-id");
config.setFunctionVersion("v1");
config.setInstanceId("test-instance-id");
config.setMaxBufferedTuples(1000);
- config.setFunctionConfig(fnConfig);
+ config.setFunctionDetails(functionDetails);
mockClient = mock(PulsarClientImpl.class);
@@ -253,46 +253,46 @@ public class JavaInstanceRunnableProcessTest {
ClassLoader clsLoader = JavaInstanceRunnableTest.class.getClassLoader();
when(fnCache.getClassLoader(anyString()))
- .thenReturn(clsLoader);
+ .thenReturn(clsLoader);
// mock producer & consumer
when(mockClient.createProducer(anyString(), any(ProducerConfiguration.class)))
- .thenAnswer(invocationOnMock -> {
- String topic = invocationOnMock.getArgumentAt(0, String.class);
- ProducerConfiguration conf = invocationOnMock.getArgumentAt(1, ProducerConfiguration.class);
- String producerName = conf.getProducerName();
-
- Pair<String, String> pair = Pair.of(topic, producerName);
- ProducerInstance producerInstance = mockProducers.get(pair);
- if (null == producerInstance) {
- Producer producer = mock(Producer.class);
- LinkedBlockingQueue<Message> msgQueue = new LinkedBlockingQueue<>();
- final ProducerInstance instance = new ProducerInstance(producer, msgQueue);
- producerInstance = instance;
- when(producer.getProducerName())
- .thenReturn(producerName);
- when(producer.getTopic())
- .thenReturn(topic);
- when(producer.sendAsync(any(Message.class)))
- .thenAnswer(invocationOnMock1 -> {
- Message msg = invocationOnMock1.getArgumentAt(0, Message.class);
- log.info("producer send message {}", msg);
-
- CompletableFuture<MessageId> future = new CompletableFuture<>();
- instance.addSendFuture(future);
- msgQueue.put(msg);
- return future;
- });
- when(producer.closeAsync()).thenReturn(FutureUtils.Void());
-
- mockProducers.put(pair, producerInstance);
- }
- return producerInstance.getProducer();
- });
+ .thenAnswer(invocationOnMock -> {
+ String topic = invocationOnMock.getArgumentAt(0, String.class);
+ ProducerConfiguration conf = invocationOnMock.getArgumentAt(1, ProducerConfiguration.class);
+ String producerName = conf.getProducerName();
+
+ Pair<String, String> pair = Pair.of(topic, producerName);
+ ProducerInstance producerInstance = mockProducers.get(pair);
+ if (null == producerInstance) {
+ Producer producer = mock(Producer.class);
+ LinkedBlockingQueue<Message> msgQueue = new LinkedBlockingQueue<>();
+ final ProducerInstance instance = new ProducerInstance(producer, msgQueue);
+ producerInstance = instance;
+ when(producer.getProducerName())
+ .thenReturn(producerName);
+ when(producer.getTopic())
+ .thenReturn(topic);
+ when(producer.sendAsync(any(Message.class)))
+ .thenAnswer(invocationOnMock1 -> {
+ Message msg = invocationOnMock1.getArgumentAt(0, Message.class);
+ log.info("producer send message {}", msg);
+
+ CompletableFuture<MessageId> future = new CompletableFuture<>();
+ instance.addSendFuture(future);
+ msgQueue.put(msg);
+ return future;
+ });
+ when(producer.closeAsync()).thenReturn(FutureUtils.Void());
+
+ mockProducers.put(pair, producerInstance);
+ }
+ return producerInstance.getProducer();
+ });
when(mockClient.subscribe(
- anyString(),
- anyString(),
- any(ConsumerConfiguration.class)
+ anyString(),
+ anyString(),
+ any(ConsumerConfiguration.class)
)).thenAnswer(invocationOnMock -> {
String topic = invocationOnMock.getArgumentAt(0, String.class);
String subscription = invocationOnMock.getArgumentAt(1, String.class);
@@ -308,26 +308,26 @@ public class JavaInstanceRunnableProcessTest {
when(consumer.getTopic()).thenReturn(topic);
when(consumer.getSubscription()).thenReturn(subscription);
when(consumer.acknowledgeAsync(any(Message.class)))
- .thenAnswer(invocationOnMock1 -> {
- Message msg = invocationOnMock1.getArgumentAt(0, Message.class);
- log.info("Ack message {} : message id = {}", msg, msg.getMessageId());
+ .thenAnswer(invocationOnMock1 -> {
+ Message msg = invocationOnMock1.getArgumentAt(0, Message.class);
+ log.info("Ack message {} : message id = {}", msg, msg.getMessageId());
- instance.removeMessage(msg.getMessageId());
- return FutureUtils.Void();
- });
+ instance.removeMessage(msg.getMessageId());
+ return FutureUtils.Void();
+ });
when(consumer.acknowledgeCumulativeAsync(any(Message.class)))
- .thenAnswer(invocationOnMock1 -> {
- Message msg = invocationOnMock1.getArgumentAt(0, Message.class);
- log.info("Ack message cumulatively message id = {}", msg, msg.getMessageId());
+ .thenAnswer(invocationOnMock1 -> {
+ Message msg = invocationOnMock1.getArgumentAt(0, Message.class);
+ log.info("Ack message cumulatively message id = {}", msg, msg.getMessageId());
- instance.removeMessagesBefore(msg.getMessageId());
- return FutureUtils.Void();
- });
+ instance.removeMessagesBefore(msg.getMessageId());
+ return FutureUtils.Void();
+ });
when(consumer.closeAsync())
- .thenAnswer(invocationOnMock1 -> {
- mockConsumers.remove(pair, instance);
- return FutureUtils.Void();
- });
+ .thenAnswer(invocationOnMock1 -> {
+ mockConsumers.remove(pair, instance);
+ return FutureUtils.Void();
+ });
doAnswer(invocationOnMock1 -> {
mockConsumers.remove(pair, instance);
return null;
@@ -353,11 +353,11 @@ public class JavaInstanceRunnableProcessTest {
PowerMockito.mockStatic(StorageClientBuilder.class);
PowerMockito.when(
- StorageClientBuilder.newBuilder()
+ StorageClientBuilder.newBuilder()
).thenReturn(mockBuilder);
when(adminClient.getStream(anyString(), anyString())).thenReturn(FutureUtils.value(
- StreamProperties.newBuilder().build()));
+ StreamProperties.newBuilder().build()));
mockTable = mock(Table.class);
when(mockStorageClient.openTable(anyString())).thenReturn(FutureUtils.value(mockTable));
@@ -367,38 +367,38 @@ public class JavaInstanceRunnableProcessTest {
mockFunctionStats = spy(new FunctionStats());
PowerMockito.whenNew(FunctionStats.class)
- .withNoArguments()
- .thenReturn(mockFunctionStats);
+ .withNoArguments()
+ .thenReturn(mockFunctionStats);
// Mock message builder
PowerMockito.mockStatic(MessageBuilder.class);
PowerMockito.when(MessageBuilder.create())
- .thenAnswer(invocationOnMock -> {
-
- Message msg = mock(Message.class);
- MessageBuilder builder = mock(MessageBuilder.class);
- when(builder.setContent(any(byte[].class)))
- .thenAnswer(invocationOnMock1 -> {
- byte[] content = invocationOnMock1.getArgumentAt(0, byte[].class);
- when(msg.getData()).thenReturn(content);
- return builder;
- });
- when(builder.setSequenceId(anyLong()))
- .thenAnswer(invocationOnMock1 -> {
- long seqId = invocationOnMock1.getArgumentAt(0, long.class);
- when(msg.getSequenceId()).thenReturn(seqId);
- return builder;
- });
- when(builder.setProperty(anyString(), anyString()))
- .thenAnswer(invocationOnMock1 -> {
- String key = invocationOnMock1.getArgumentAt(0, String.class);
- String value = invocationOnMock1.getArgumentAt(1, String.class);
- when(msg.getProperty(eq(key))).thenReturn(value);
- return builder;
+ .thenAnswer(invocationOnMock -> {
+
+ Message msg = mock(Message.class);
+ MessageBuilder builder = mock(MessageBuilder.class);
+ when(builder.setContent(any(byte[].class)))
+ .thenAnswer(invocationOnMock1 -> {
+ byte[] content = invocationOnMock1.getArgumentAt(0, byte[].class);
+ when(msg.getData()).thenReturn(content);
+ return builder;
});
- when(builder.build()).thenReturn(msg);
- return builder;
- });
+ when(builder.setSequenceId(anyLong()))
+ .thenAnswer(invocationOnMock1 -> {
+ long seqId = invocationOnMock1.getArgumentAt(0, long.class);
+ when(msg.getSequenceId()).thenReturn(seqId);
+ return builder;
+ });
+ when(builder.setProperty(anyString(), anyString()))
+ .thenAnswer(invocationOnMock1 -> {
+ String key = invocationOnMock1.getArgumentAt(0, String.class);
+ String value = invocationOnMock1.getArgumentAt(1, String.class);
+ when(msg.getProperty(eq(key))).thenReturn(value);
+ return builder;
+ });
+ when(builder.build()).thenReturn(msg);
+ return builder;
+ });
}
/**
@@ -407,11 +407,11 @@ public class JavaInstanceRunnableProcessTest {
@Test
public void testSetupJavaInstance() throws Exception {
JavaInstanceRunnable runnable = new JavaInstanceRunnable(
- config,
- fnCache,
- "test-jar-file",
- mockClient,
- TEST_STORAGE_SERVICE_URL);
+ config,
+ fnCache,
+ "test-jar-file",
+ mockClient,
+ TEST_STORAGE_SERVICE_URL);
runnable.setupJavaInstance();
@@ -419,17 +419,17 @@ public class JavaInstanceRunnableProcessTest {
// 1. verify jar is loaded
verify(fnCache, times(1))
- .registerFunctionInstance(
- eq(config.getFunctionId()),
- eq(config.getInstanceId()),
- eq(Arrays.asList("test-jar-file")),
- eq(Collections.emptyList())
- );
+ .registerFunctionInstance(
+ eq(config.getFunctionId()),
+ eq(config.getInstanceId()),
+ eq(Arrays.asList("test-jar-file")),
+ eq(Collections.emptyList())
+ );
verify(fnCache, times(1))
- .getClassLoader(eq(config.getFunctionId()));
+ .getClassLoader(eq(config.getFunctionId()));
// 2. verify serde is setup
- for (String inputTopic : fnConfig.getInputsList()) {
+ for (String inputTopic : functionDetails.getInputsList()) {
assertTrue(runnable.getInputSerDe().containsKey(inputTopic));
assertTrue(runnable.getInputSerDe().get(inputTopic) instanceof DefaultSerDe);
DefaultSerDe serDe = (DefaultSerDe) runnable.getInputSerDe().get(inputTopic);
@@ -440,8 +440,8 @@ public class JavaInstanceRunnableProcessTest {
MessageProcessor processor = runnable.getProcessor();
assertTrue(processor instanceof AtLeastOnceProcessor);
assertSame(mockProducers.get(Pair.of(
- fnConfig.getOutput(),
- null
+ functionDetails.getOutput(),
+ null
)).getProducer(), ((AtLeastOnceProcessor) processor).getProducer());
assertEquals(mockConsumers.size(), processor.getInputConsumers().size());
@@ -449,8 +449,8 @@ public class JavaInstanceRunnableProcessTest {
String topic = consumerEntry.getKey();
Consumer mockConsumer = mockConsumers.get(Pair.of(
- topic,
- FunctionConfigUtils.getFullyQualifiedName(fnConfig))).getConsumer();
+ topic,
+ FunctionDetailsUtils.getFullyQualifiedName(functionDetails))).getConsumer();
assertSame(mockConsumer, consumerEntry.getValue());
}
@@ -475,32 +475,32 @@ public class JavaInstanceRunnableProcessTest {
// function is unregistered
verify(fnCache, times(1)).unregisterFunctionInstance(
- eq(config.getFunctionId()), eq(config.getInstanceId()));
+ eq(config.getFunctionId()), eq(config.getInstanceId()));
}
@Test
public void testAtMostOnceProcessing() throws Exception {
- FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig)
- .setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE)
- .build();
- config.setFunctionConfig(newFnConfig);
+ FunctionDetails newFunctionDetails = FunctionDetails.newBuilder(functionDetails)
+ .setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE)
+ .build();
+ config.setFunctionDetails(newFunctionDetails);
@Cleanup("shutdown")
ExecutorService executorService = Executors.newSingleThreadExecutor();
try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(
- config,
- fnCache,
- "test-jar-file",
- mockClient,
- null)) {
+ config,
+ fnCache,
+ "test-jar-file",
+ mockClient,
+ null)) {
executorService.submit(runnable);
Pair<String, String> consumerId = Pair.of(
- newFnConfig.getInputs(0),
- FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
+ newFunctionDetails.getInputs(0),
+ FunctionDetailsUtils.getFullyQualifiedName(newFunctionDetails));
ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
while (null == consumerInstance) {
TimeUnit.MILLISECONDS.sleep(20);
@@ -514,10 +514,10 @@ public class JavaInstanceRunnableProcessTest {
Message msg = mock(Message.class);
when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
when(msg.getMessageId())
- .thenReturn(new MessageIdImpl(1L, i, 0));
+ .thenReturn(new MessageIdImpl(1L, i, 0));
consumerInstance.addMessage(msg);
consumerInstance.getConf().getMessageListener()
- .received(consumerInstance.getConsumer(), msg);
+ .received(consumerInstance.getConsumer(), msg);
}
// wait until all the messages are published
@@ -531,7 +531,7 @@ public class JavaInstanceRunnableProcessTest {
// verify acknowledge before send completes
verify(consumerInstance.getConsumer(), times(10))
- .acknowledgeAsync(any(Message.class));
+ .acknowledgeAsync(any(Message.class));
assertEquals(0, consumerInstance.getNumMessages());
// complete all the publishes
@@ -543,33 +543,33 @@ public class JavaInstanceRunnableProcessTest {
// acknowledges count should remain same
verify(consumerInstance.getConsumer(), times(10))
- .acknowledgeAsync(any(Message.class));
+ .acknowledgeAsync(any(Message.class));
}
}
@Test
public void testAtMostOnceProcessingFailures() throws Exception {
- FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig)
- .setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE)
- .setClassName(TestFailureFunction.class.getName())
- .build();
- config.setFunctionConfig(newFnConfig);
+ FunctionDetails newFunctionDetails = FunctionDetails.newBuilder(functionDetails)
+ .setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE)
+ .setClassName(TestFailureFunction.class.getName())
+ .build();
+ config.setFunctionDetails(newFunctionDetails);
@Cleanup("shutdown")
ExecutorService executorService = Executors.newSingleThreadExecutor();
try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(
- config,
- fnCache,
- "test-jar-file",
- mockClient,
- null)) {
+ config,
+ fnCache,
+ "test-jar-file",
+ mockClient,
+ null)) {
executorService.submit(runnable);
Pair<String, String> consumerId = Pair.of(
- newFnConfig.getInputs(0),
- FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
+ newFunctionDetails.getInputs(0),
+ FunctionDetailsUtils.getFullyQualifiedName(newFunctionDetails));
ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
while (null == consumerInstance) {
TimeUnit.MILLISECONDS.sleep(20);
@@ -583,10 +583,10 @@ public class JavaInstanceRunnableProcessTest {
Message msg = mock(Message.class);
when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
when(msg.getMessageId())
- .thenReturn(new MessageIdImpl(1L, i, 0));
+ .thenReturn(new MessageIdImpl(1L, i, 0));
consumerInstance.addMessage(msg);
consumerInstance.getConf().getMessageListener()
- .received(consumerInstance.getConsumer(), msg);
+ .received(consumerInstance.getConsumer(), msg);
}
// wait until all the messages are published
@@ -604,7 +604,7 @@ public class JavaInstanceRunnableProcessTest {
// verify acknowledge before send completes
verify(consumerInstance.getConsumer(), times(10))
- .acknowledgeAsync(any(Message.class));
+ .acknowledgeAsync(any(Message.class));
assertEquals(0, consumerInstance.getNumMessages());
// complete all the publishes
@@ -616,33 +616,33 @@ public class JavaInstanceRunnableProcessTest {
// acknowledges count should remain same
verify(consumerInstance.getConsumer(), times(10))
- .acknowledgeAsync(any(Message.class));
+ .acknowledgeAsync(any(Message.class));
assertEquals(0, consumerInstance.getNumMessages());
}
}
@Test
public void testAtLeastOnceProcessing() throws Exception {
- FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig)
- .setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE)
- .build();
- config.setFunctionConfig(newFnConfig);
+ FunctionDetails newFunctionDetails = FunctionDetails.newBuilder(functionDetails)
+ .setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE)
+ .build();
+ config.setFunctionDetails(newFunctionDetails);
@Cleanup("shutdown")
ExecutorService executorService = Executors.newSingleThreadExecutor();
try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(
- config,
- fnCache,
- "test-jar-file",
- mockClient,
- null)) {
+ config,
+ fnCache,
+ "test-jar-file",
+ mockClient,
+ null)) {
executorService.submit(runnable);
Pair<String, String> consumerId = Pair.of(
- newFnConfig.getInputs(0),
- FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
+ newFunctionDetails.getInputs(0),
+ FunctionDetailsUtils.getFullyQualifiedName(newFunctionDetails));
ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
while (null == consumerInstance) {
TimeUnit.MILLISECONDS.sleep(20);
@@ -656,10 +656,10 @@ public class JavaInstanceRunnableProcessTest {
Message msg = mock(Message.class);
when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
when(msg.getMessageId())
- .thenReturn(new MessageIdImpl(1L, i, 0));
+ .thenReturn(new MessageIdImpl(1L, i, 0));
consumerInstance.addMessage(msg);
consumerInstance.getConf().getMessageListener()
- .received(consumerInstance.getConsumer(), msg);
+ .received(consumerInstance.getConsumer(), msg);
}
// wait until all the messages are published
@@ -673,7 +673,7 @@ public class JavaInstanceRunnableProcessTest {
// verify acknowledge before send completes
verify(consumerInstance.getConsumer(), times(0))
- .acknowledgeAsync(any(Message.class));
+ .acknowledgeAsync(any(Message.class));
assertEquals(10, consumerInstance.getNumMessages());
// complete all the publishes
@@ -685,34 +685,34 @@ public class JavaInstanceRunnableProcessTest {
// acknowledges count should remain same
verify(consumerInstance.getConsumer(), times(10))
- .acknowledgeAsync(any(Message.class));
+ .acknowledgeAsync(any(Message.class));
assertEquals(0, consumerInstance.getNumMessages());
}
}
@Test
public void testAtLeastOnceProcessingFailures() throws Exception {
- FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig)
- .setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE)
- .setClassName(TestFailureFunction.class.getName())
- .build();
- config.setFunctionConfig(newFnConfig);
+ FunctionDetails newFunctionDetails = FunctionDetails.newBuilder(functionDetails)
+ .setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE)
+ .setClassName(TestFailureFunction.class.getName())
+ .build();
+ config.setFunctionDetails(newFunctionDetails);
@Cleanup("shutdown")
ExecutorService executorService = Executors.newSingleThreadExecutor();
try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(
- config,
- fnCache,
- "test-jar-file",
- mockClient,
- null)) {
+ config,
+ fnCache,
+ "test-jar-file",
+ mockClient,
+ null)) {
executorService.submit(runnable);
Pair<String, String> consumerId = Pair.of(
- newFnConfig.getInputs(0),
- FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
+ newFunctionDetails.getInputs(0),
+ FunctionDetailsUtils.getFullyQualifiedName(newFunctionDetails));
ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
while (null == consumerInstance) {
TimeUnit.MILLISECONDS.sleep(20);
@@ -726,10 +726,10 @@ public class JavaInstanceRunnableProcessTest {
Message msg = mock(Message.class);
when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
when(msg.getMessageId())
- .thenReturn(new MessageIdImpl(1L, i, 0));
+ .thenReturn(new MessageIdImpl(1L, i, 0));
consumerInstance.addMessage(msg);
consumerInstance.getConf().getMessageListener()
- .received(consumerInstance.getConsumer(), msg);
+ .received(consumerInstance.getConsumer(), msg);
}
// wait until all the messages are published
@@ -747,7 +747,7 @@ public class JavaInstanceRunnableProcessTest {
// verify acknowledge before send completes
verify(consumerInstance.getConsumer(), times(0))
- .acknowledgeAsync(any(Message.class));
+ .acknowledgeAsync(any(Message.class));
assertEquals(10, consumerInstance.getNumMessages());
// complete all the publishes
@@ -759,38 +759,38 @@ public class JavaInstanceRunnableProcessTest {
// only 5 succeed messages are acknowledged
verify(consumerInstance.getConsumer(), times(5))
- .acknowledgeAsync(any(Message.class));
+ .acknowledgeAsync(any(Message.class));
assertEquals(5, consumerInstance.getNumMessages());
for (int i = 0; i < 10; i++) {
assertEquals(
- i % 2 == 0,
- consumerInstance.containMessage(new MessageIdImpl(1L, i, 0)));
+ i % 2 == 0,
+ consumerInstance.containMessage(new MessageIdImpl(1L, i, 0)));
}
}
}
@Test
public void testEffectivelyOnceProcessing() throws Exception {
- FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig)
- .setProcessingGuarantees(ProcessingGuarantees.EFFECTIVELY_ONCE)
- .build();
- config.setFunctionConfig(newFnConfig);
+ FunctionDetails newFunctionDetails = FunctionDetails.newBuilder(functionDetails)
+ .setProcessingGuarantees(ProcessingGuarantees.EFFECTIVELY_ONCE)
+ .build();
+ config.setFunctionDetails(newFunctionDetails);
@Cleanup("shutdown")
ExecutorService executorService = Executors.newSingleThreadExecutor();
try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(
- config,
- fnCache,
- "test-jar-file",
- mockClient,
- null)) {
+ config,
+ fnCache,
+ "test-jar-file",
+ mockClient,
+ null)) {
executorService.submit(runnable);
Pair<String, String> consumerId = Pair.of(
- newFnConfig.getInputs(0),
- FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
+ newFunctionDetails.getInputs(0),
+ FunctionDetailsUtils.getFullyQualifiedName(newFunctionDetails));
ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
while (null == consumerInstance) {
TimeUnit.MILLISECONDS.sleep(20);
@@ -802,10 +802,10 @@ public class JavaInstanceRunnableProcessTest {
Message msg = mock(Message.class);
when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
when(msg.getMessageId())
- .thenReturn(new MessageIdImpl(1L, i, 0));
+ .thenReturn(new MessageIdImpl(1L, i, 0));
consumerInstance.addMessage(msg);
consumerInstance.getConf().getMessageListener()
- .received(consumerInstance.getConsumer(), msg);
+ .received(consumerInstance.getConsumer(), msg);
}
ProducerInstance producerInstance;
@@ -821,14 +821,14 @@ public class JavaInstanceRunnableProcessTest {
assertEquals("message-" + i + "!", new String(msg.getData(), UTF_8));
// sequence id is not set for AT_MOST_ONCE processing
assertEquals(
- Utils.getSequenceId(
- new MessageIdImpl(1L, i, 0)),
- msg.getSequenceId());
+ Utils.getSequenceId(
+ new MessageIdImpl(1L, i, 0)),
+ msg.getSequenceId());
}
// verify acknowledge before send completes
verify(consumerInstance.getConsumer(), times(0))
- .acknowledgeCumulativeAsync(any(Message.class));
+ .acknowledgeCumulativeAsync(any(Message.class));
assertEquals(10, consumerInstance.getNumMessages());
// complete all the publishes
@@ -840,34 +840,34 @@ public class JavaInstanceRunnableProcessTest {
// acknowledges count should remain same
verify(consumerInstance.getConsumer(), times(10))
- .acknowledgeCumulativeAsync(any(Message.class));
+ .acknowledgeCumulativeAsync(any(Message.class));
assertEquals(0, consumerInstance.getNumMessages());
}
}
@Test
public void testEffectivelyOnceProcessingFailures() throws Exception {
- FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig)
- .setProcessingGuarantees(ProcessingGuarantees.EFFECTIVELY_ONCE)
- .setClassName(TestFailureFunction.class.getName())
- .build();
- config.setFunctionConfig(newFnConfig);
+ FunctionDetails newFunctionDetails = FunctionDetails.newBuilder(functionDetails)
+ .setProcessingGuarantees(ProcessingGuarantees.EFFECTIVELY_ONCE)
+ .setClassName(TestFailureFunction.class.getName())
+ .build();
+ config.setFunctionDetails(newFunctionDetails);
@Cleanup("shutdown")
ExecutorService executorService = Executors.newSingleThreadExecutor();
try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(
- config,
- fnCache,
- "test-jar-file",
- mockClient,
- null)) {
+ config,
+ fnCache,
+ "test-jar-file",
+ mockClient,
+ null)) {
executorService.submit(runnable);
Pair<String, String> consumerId = Pair.of(
- newFnConfig.getInputs(0),
- FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
+ newFunctionDetails.getInputs(0),
+ FunctionDetailsUtils.getFullyQualifiedName(newFunctionDetails));
ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
while (null == consumerInstance) {
TimeUnit.MILLISECONDS.sleep(20);
@@ -880,13 +880,13 @@ public class JavaInstanceRunnableProcessTest {
Message msg = mock(Message.class);
when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
when(msg.getMessageId())
- .thenReturn(new MessageIdImpl(1L, i, 0));
+ .thenReturn(new MessageIdImpl(1L, i, 0));
msgs[i-1] = msg;
consumerInstance.addMessage(msg);
consumerInstance.getConf().getMessageListener()
- .received(consumerInstance.getConsumer(), msg);
+ .received(consumerInstance.getConsumer(), msg);
}
ProducerInstance producerInstance;
@@ -899,15 +899,15 @@ public class JavaInstanceRunnableProcessTest {
Message msg = producerInstance.msgQueue.take();
assertEquals("message-1!", new String(msg.getData(), UTF_8));
assertEquals(
- Utils.getSequenceId(
- new MessageIdImpl(1L, 1, 0)),
- msg.getSequenceId());
+ Utils.getSequenceId(
+ new MessageIdImpl(1L, 1, 0)),
+ msg.getSequenceId());
assertNull(producerInstance.msgQueue.poll());
// the first result message is sent but the send future is not completed yet
// so no acknowledge would happen
verify(consumerInstance.getConsumer(), times(0))
- .acknowledgeCumulativeAsync(any(Message.class));
+ .acknowledgeCumulativeAsync(any(Message.class));
// since the second message failed to process, for correctness, the instance
// will close the existing consumer and resubscribe
@@ -920,21 +920,21 @@ public class JavaInstanceRunnableProcessTest {
Message secondMsg = mock(Message.class);
when(secondMsg.getData()).thenReturn("message-2".getBytes(UTF_8));
when(secondMsg.getMessageId())
- .thenReturn(new MessageIdImpl(1L, 2, 0));
+ .thenReturn(new MessageIdImpl(1L, 2, 0));
secondInstance.addMessage(secondMsg);
secondInstance.getConf().getMessageListener()
- .received(secondInstance.getConsumer(), secondMsg);
+ .received(secondInstance.getConsumer(), secondMsg);
Message secondReceivedMsg = producerInstance.msgQueue.take();
assertEquals("message-2!", new String(secondReceivedMsg.getData(), UTF_8));
assertEquals(
- Utils.getSequenceId(
- new MessageIdImpl(1L, 2, 0)),
- secondReceivedMsg.getSequenceId());
+ Utils.getSequenceId(
+ new MessageIdImpl(1L, 2, 0)),
+ secondReceivedMsg.getSequenceId());
// the first result message is sent
verify(secondInstance.getConsumer(), times(0))
- .acknowledgeCumulativeAsync(any(Message.class));
+ .acknowledgeCumulativeAsync(any(Message.class));
// complete all the publishes
synchronized (producerInstance) {
@@ -946,41 +946,41 @@ public class JavaInstanceRunnableProcessTest {
// all 2 messages are sent
verify(consumerInstance.getConsumer(), times(1))
- .acknowledgeCumulativeAsync(same(msgs[0]));
+ .acknowledgeCumulativeAsync(same(msgs[0]));
verify(consumerInstance.getConsumer(), times(0))
- .acknowledgeCumulativeAsync(same(msgs[1]));
+ .acknowledgeCumulativeAsync(same(msgs[1]));
verify(consumerInstance.getConsumer(), times(0))
- .acknowledgeCumulativeAsync(same(secondMsg));
+ .acknowledgeCumulativeAsync(same(secondMsg));
verify(secondInstance.getConsumer(), times(0))
- .acknowledgeCumulativeAsync(same(msgs[0]));
+ .acknowledgeCumulativeAsync(same(msgs[0]));
verify(secondInstance.getConsumer(), times(0))
- .acknowledgeCumulativeAsync(same(msgs[1]));
+ .acknowledgeCumulativeAsync(same(msgs[1]));
}
}
@Test
public void testVoidFunction() throws Exception {
- FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig)
- .setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE)
- .setClassName(TestVoidFunction.class.getName())
- .build();
- config.setFunctionConfig(newFnConfig);
+ FunctionDetails newFunctionDetails = FunctionDetails.newBuilder(functionDetails)
+ .setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE)
+ .setClassName(TestVoidFunction.class.getName())
+ .build();
+ config.setFunctionDetails(newFunctionDetails);
@Cleanup("shutdown")
ExecutorService executorService = Executors.newSingleThreadExecutor();
try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(
- config,
- fnCache,
- "test-jar-file",
- mockClient,
- null)) {
+ config,
+ fnCache,
+ "test-jar-file",
+ mockClient,
+ null)) {
executorService.submit(runnable);
Pair<String, String> consumerId = Pair.of(
- newFnConfig.getInputs(0),
- FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
+ newFunctionDetails.getInputs(0),
+ FunctionDetailsUtils.getFullyQualifiedName(newFunctionDetails));
ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
while (null == consumerInstance) {
TimeUnit.MILLISECONDS.sleep(20);
@@ -992,10 +992,10 @@ public class JavaInstanceRunnableProcessTest {
Message msg = mock(Message.class);
when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
when(msg.getMessageId())
- .thenReturn(new MessageIdImpl(1L, i, 0));
+ .thenReturn(new MessageIdImpl(1L, i, 0));
consumerInstance.addMessage(msg);
consumerInstance.getConf().getMessageListener()
- .received(consumerInstance.getConsumer(), msg);
+ .received(consumerInstance.getConsumer(), msg);
}
// wait until all the messages are published
@@ -1009,4 +1009,4 @@ public class JavaInstanceRunnableProcessTest {
assertTrue(mockProducers.isEmpty());
}
}
-}
+}
\ No newline at end of file
diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java
index 1e9781c..a1672ba 100644
--- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java
+++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java
@@ -25,7 +25,7 @@ import org.apache.pulsar.functions.api.Context;
import org.apache.pulsar.functions.api.Function;
import org.apache.pulsar.functions.api.SerDe;
import org.apache.pulsar.functions.api.utils.DefaultSerDe;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails;
import org.testng.annotations.Test;
import java.lang.reflect.InvocationTargetException;
@@ -48,17 +48,17 @@ public class JavaInstanceRunnableTest {
}
private static InstanceConfig createInstanceConfig(boolean addCustom, String outputSerde) {
- FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder();
+ FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
if (!addCustom) {
- functionConfigBuilder.addInputs("TEST");
+ functionDetailsBuilder.addInputs("TEST");
} else {
- functionConfigBuilder.putCustomSerdeInputs("TEST", IntegerSerDe.class.getName());
+ functionDetailsBuilder.putCustomSerdeInputs("TEST", IntegerSerDe.class.getName());
}
if (outputSerde != null) {
- functionConfigBuilder.setOutputSerdeClassName(outputSerde);
+ functionDetailsBuilder.setOutputSerdeClassName(outputSerde);
}
InstanceConfig instanceConfig = new InstanceConfig();
- instanceConfig.setFunctionConfig(functionConfigBuilder.build());
+ instanceConfig.setFunctionDetails(functionDetailsBuilder.build());
instanceConfig.setMaxBufferedTuples(1024);
return instanceConfig;
}
diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java
index 53c93a4..c6f7577 100644
--- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java
+++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java
@@ -24,7 +24,7 @@ import static org.testng.Assert.assertNotNull;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.functions.api.Function;
import org.apache.pulsar.functions.api.utils.DefaultSerDe;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails;
import org.testng.annotations.Test;
import java.util.HashMap;
@@ -32,11 +32,11 @@ import java.util.HashMap;
public class JavaInstanceTest {
private static InstanceConfig createInstanceConfig() {
- FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder();
- functionConfigBuilder.addInputs("TEST");
- functionConfigBuilder.setOutputSerdeClassName(DefaultSerDe.class.getName());
+ FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
+ functionDetailsBuilder.addInputs("TEST");
+ functionDetailsBuilder.setOutputSerdeClassName(DefaultSerDe.class.getName());
InstanceConfig instanceConfig = new InstanceConfig();
- instanceConfig.setFunctionConfig(functionConfigBuilder.build());
+ instanceConfig.setFunctionDetails(functionDetailsBuilder.build());
return instanceConfig;
}
diff --git a/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/MetricsSink.java b/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/MetricsSink.java
index 488edf5..d22bf6a 100644
--- a/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/MetricsSink.java
+++ b/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/MetricsSink.java
@@ -42,11 +42,10 @@ public interface MetricsSink extends AutoCloseable {
/**
* Process a metrics record in the sink
- *
- * @param record the record to put
- * @param functionConfig functionConfig that generated this record
+ * @param record the record to put
+ * @param functionDetails functionDetails that generated this record
*/
- void processRecord(MetricsData record, Function.FunctionConfig functionConfig);
+ void processRecord(MetricsData record, Function.FunctionDetails functionDetails);
/**
* Flush any buffered metrics
diff --git a/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/FileSink.java b/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/FileSink.java
index 06e0cb2..64bc97d 100644
--- a/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/FileSink.java
+++ b/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/FileSink.java
@@ -29,7 +29,7 @@ import java.util.logging.Logger;
import org.apache.pulsar.functions.proto.Function;
import org.apache.pulsar.functions.proto.InstanceCommunication;
import org.apache.pulsar.functions.metrics.MetricsSink;
-import org.apache.pulsar.functions.utils.FunctionConfigUtils;
+import org.apache.pulsar.functions.utils.FunctionDetailsUtils;
import org.apache.pulsar.functions.utils.Utils;
/**
@@ -82,9 +82,9 @@ public class FileSink implements MetricsSink {
}
@Override
- public void processRecord(InstanceCommunication.MetricsData record, Function.FunctionConfig functionConfig) {
+ public void processRecord(InstanceCommunication.MetricsData record, Function.FunctionDetails FunctionDetails) {
if (isFileStart) {
- String filenamePrefix = filenameKey + "." + FunctionConfigUtils.getFullyQualifiedName(functionConfig);
+ String filenamePrefix = filenameKey + "." + FunctionDetailsUtils.getFullyQualifiedName(FunctionDetails);
writer = openNewFile(String.format("%s.%d", filenamePrefix, currentFileIndex));
writer.print("[");
diff --git a/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/PrometheusSink.java b/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/PrometheusSink.java
index 65a24ca..26aa940 100644
--- a/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/PrometheusSink.java
+++ b/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/PrometheusSink.java
@@ -29,7 +29,7 @@ import com.google.common.cache.Cache;
import lombok.Getter;
import org.apache.pulsar.functions.proto.Function;
import org.apache.pulsar.functions.proto.InstanceCommunication.MetricsData;
-import org.apache.pulsar.functions.utils.FunctionConfigUtils;
+import org.apache.pulsar.functions.utils.FunctionDetailsUtils;
/**
* A web sink that exposes and endpoint that Prometheus can scrape
@@ -67,9 +67,9 @@ public class PrometheusSink extends AbstractWebSink {
final StringBuilder sb = new StringBuilder();
metrics.forEach((String source, Map<String, Double> sourceMetrics) -> {
- String tenant = FunctionConfigUtils.extractTenantFromFQN(source);
- String namespace = FunctionConfigUtils.extractNamespaceFromFQN(source);
- String name = FunctionConfigUtils.extractFunctionNameFromFQN(source);
+ String tenant = FunctionDetailsUtils.extractTenantFromFQN(source);
+ String namespace = FunctionDetailsUtils.extractNamespaceFromFQN(source);
+ String name = FunctionDetailsUtils.extractFunctionNameFromFQN(source);
sourceMetrics.forEach((String metricName, Double value) -> {
@@ -91,8 +91,8 @@ public class PrometheusSink extends AbstractWebSink {
}
@Override
- public void processRecord(MetricsData record, Function.FunctionConfig functionConfig) {
- final String source = FunctionConfigUtils.getFullyQualifiedName(functionConfig);
+ public void processRecord(MetricsData record, Function.FunctionDetails functionDetails) {
+ final String source = FunctionDetailsUtils.getFullyQualifiedName(functionDetails);
Map<String, Double> sourceCache = metricsCache.getIfPresent(source);
if (sourceCache == null) {
diff --git a/pulsar-functions/metrics/src/test/java/org/apache/pulsar/functions/metrics/sink/PrometheusSinkTests.java b/pulsar-functions/metrics/src/test/java/org/apache/pulsar/functions/metrics/sink/PrometheusSinkTests.java
index 63bb579..887427f 100644
--- a/pulsar-functions/metrics/src/test/java/org/apache/pulsar/functions/metrics/sink/PrometheusSinkTests.java
+++ b/pulsar-functions/metrics/src/test/java/org/apache/pulsar/functions/metrics/sink/PrometheusSinkTests.java
@@ -28,7 +28,7 @@ import java.util.Set;
import org.apache.pulsar.functions.proto.Function;
import org.apache.pulsar.functions.proto.InstanceCommunication;
-import org.apache.pulsar.functions.utils.FunctionConfigUtils;
+import org.apache.pulsar.functions.utils.FunctionDetailsUtils;
import org.testng.annotations.Test;
import org.testng.annotations.BeforeMethod;
import static org.testng.Assert.assertEquals;
@@ -86,29 +86,29 @@ public class PrometheusSinkTests {
public void testMetricsGrouping() {
PrometheusTestSink sink = new PrometheusTestSink();
sink.init(defaultConf);
- Function.FunctionConfig functionConfig = createFunctionConfig("tenant", "namespace", "functionname");
- sink.processRecord(records, functionConfig);
+ Function.FunctionDetails functionDetails = createFunctionDetails("tenant", "namespace", "functionname");
+ sink.processRecord(records, functionDetails);
final Map<String, Map<String, Double>> metrics = sink.getMetrics();
- assertTrue(metrics.containsKey(FunctionConfigUtils.getFullyQualifiedName(functionConfig)));
+ assertTrue(metrics.containsKey(FunctionDetailsUtils.getFullyQualifiedName(functionDetails)));
}
@Test
public void testResponse() throws IOException {
PrometheusTestSink sink = new PrometheusTestSink();
sink.init(defaultConf);
- Function.FunctionConfig functionConfig = createFunctionConfig("tenant", "namespace", "functionname");
- sink.processRecord(records, functionConfig);
+ Function.FunctionDetails functionDetails = createFunctionDetails("tenant", "namespace", "functionname");
+ sink.processRecord(records, functionDetails);
final List<String> expectedLines = Arrays.asList(
- createMetric(functionConfig, "metric_1_count", 2),
- createMetric(functionConfig, "metric_1_sum", 5),
- createMetric(functionConfig, "metric_1_max", 3),
- createMetric(functionConfig, "metric_1_min", 2),
- createMetric(functionConfig, "metric_2_count", 3),
- createMetric(functionConfig, "metric_2_sum", 6),
- createMetric(functionConfig, "metric_2_max", 3),
- createMetric(functionConfig, "metric_2_min", 1)
+ createMetric(functionDetails, "metric_1_count", 2),
+ createMetric(functionDetails, "metric_1_sum", 5),
+ createMetric(functionDetails, "metric_1_max", 3),
+ createMetric(functionDetails, "metric_1_min", 2),
+ createMetric(functionDetails, "metric_2_count", 3),
+ createMetric(functionDetails, "metric_2_sum", 6),
+ createMetric(functionDetails, "metric_2_max", 3),
+ createMetric(functionDetails, "metric_2_min", 1)
);
final Set<String> generatedLines =
@@ -121,16 +121,16 @@ public class PrometheusSinkTests {
});
}
- private String createMetric(Function.FunctionConfig functionConfig,
+ private String createMetric(Function.FunctionDetails functionDetails,
String metric, double value) {
return String.format("pulsar_function_%s"
+ "{tenant=\"%s\",namespace=\"%s\",functionname=\"%s\"}"
+ " %s %d",
- metric, functionConfig.getTenant(), functionConfig.getNamespace(), functionConfig.getName(), value, NOW);
+ metric, functionDetails.getTenant(), functionDetails.getNamespace(), functionDetails.getName(), value, NOW);
}
- private Function.FunctionConfig createFunctionConfig(String tenant, String namespace, String name) {
- Function.FunctionConfig.Builder bldr = Function.FunctionConfig.newBuilder();
+ private Function.FunctionDetails createFunctionDetails(String tenant, String namespace, String name) {
+ Function.FunctionDetails.Builder bldr = Function.FunctionDetails.newBuilder();
bldr.setTenant(tenant);
bldr.setNamespace(namespace);
bldr.setName(name);
diff --git a/pulsar-functions/proto/src/main/proto/Function.proto b/pulsar-functions/proto/src/main/proto/Function.proto
index c12059f..c9b3da2 100644
--- a/pulsar-functions/proto/src/main/proto/Function.proto
+++ b/pulsar-functions/proto/src/main/proto/Function.proto
@@ -23,7 +23,7 @@ package proto;
option java_package = "org.apache.pulsar.functions.proto";
option java_outer_classname = "Function";
-message FunctionConfig {
+message FunctionDetails {
enum ProcessingGuarantees {
ATLEAST_ONCE = 0; // [default value]
ATMOST_ONCE = 1;
@@ -60,7 +60,7 @@ message PackageLocationMetaData {
}
message FunctionMetaData {
- FunctionConfig functionConfig = 1;
+ FunctionDetails functionDetails = 1;
PackageLocationMetaData packageLocation = 2;
uint64 version = 3;
uint64 createTime = 4;
diff --git a/pulsar-functions/proto/src/test/java/org/apache/pulsar/functions/proto/FunctionConfigTest.java b/pulsar-functions/proto/src/test/java/org/apache/pulsar/functions/proto/FunctionDetailsTest.java
similarity index 79%
rename from pulsar-functions/proto/src/test/java/org/apache/pulsar/functions/proto/FunctionConfigTest.java
rename to pulsar-functions/proto/src/test/java/org/apache/pulsar/functions/proto/FunctionDetailsTest.java
index b4cb287..c5f7dfa 100644
--- a/pulsar-functions/proto/src/test/java/org/apache/pulsar/functions/proto/FunctionConfigTest.java
+++ b/pulsar-functions/proto/src/test/java/org/apache/pulsar/functions/proto/FunctionDetailsTest.java
@@ -20,21 +20,21 @@ package org.apache.pulsar.functions.proto;
import static org.testng.Assert.assertEquals;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig.ProcessingGuarantees;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails.ProcessingGuarantees;
import org.testng.annotations.Test;
/**
- * Unit test for {@link FunctionConfig}.
+ * Unit test for {@link FunctionDetails}.
*/
-public class FunctionConfigTest {
+public class FunctionDetailsTest {
/**
* Make sure the default processing guarantee is always `ATLEAST_ONCE`.
*/
@Test
public void testDefaultProcessingGuarantee() {
- FunctionConfig fc = FunctionConfig.newBuilder().build();
+ FunctionDetails fc = FunctionDetails.newBuilder().build();
assertEquals(ProcessingGuarantees.ATLEAST_ONCE, fc.getProcessingGuarantees());
}
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java
index 594ea10..c549eab 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java
@@ -30,7 +30,7 @@ import io.grpc.ServerBuilder;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.functions.instance.InstanceConfig;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails;
import org.apache.pulsar.functions.proto.InstanceCommunication;
import org.apache.pulsar.functions.proto.InstanceControlGrpc;
@@ -74,7 +74,7 @@ public class JavaInstanceMain {
protected String logTopic;
@Parameter(names = "--processing_guarantees", description = "Processing Guarantees\n", required = true)
- protected FunctionConfig.ProcessingGuarantees processingGuarantees;
+ protected FunctionDetails.ProcessingGuarantees processingGuarantees;
@Parameter(names = "--instance_id", description = "Instance Id\n", required = true)
protected String instanceId;
@@ -114,15 +114,15 @@ public class JavaInstanceMain {
instanceConfig.setFunctionVersion(functionVersion);
instanceConfig.setInstanceId(instanceId);
instanceConfig.setMaxBufferedTuples(maxBufferedTuples);
- FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder();
- functionConfigBuilder.setTenant(tenant);
- functionConfigBuilder.setNamespace(namespace);
- functionConfigBuilder.setName(functionName);
- functionConfigBuilder.setClassName(className);
+ FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
+ functionDetailsBuilder.setTenant(tenant);
+ functionDetailsBuilder.setNamespace(namespace);
+ functionDetailsBuilder.setName(functionName);
+ functionDetailsBuilder.setClassName(className);
if (defaultSerdeInputTopics != null) {
String[] inputTopics = defaultSerdeInputTopics.split(",");
for (String inputTopic : inputTopics) {
- functionConfigBuilder.addInputs(inputTopic);
+ functionDetailsBuilder.addInputs(inputTopic);
}
}
if (customSerdeInputTopics != null && customSerdeClassnames != null) {
@@ -132,31 +132,31 @@ public class JavaInstanceMain {
throw new RuntimeException("Error specifying inputs");
}
for (int i = 0; i < inputTopics.length; ++i) {
- functionConfigBuilder.putCustomSerdeInputs(inputTopics[i], inputSerdeClassNames[i]);
+ functionDetailsBuilder.putCustomSerdeInputs(inputTopics[i], inputSerdeClassNames[i]);
}
}
if (outputSerdeClassName != null) {
- functionConfigBuilder.setOutputSerdeClassName(outputSerdeClassName);
+ functionDetailsBuilder.setOutputSerdeClassName(outputSerdeClassName);
}
if (outputTopicName != null) {
- functionConfigBuilder.setOutput(outputTopicName);
+ functionDetailsBuilder.setOutput(outputTopicName);
}
if (logTopic != null) {
- functionConfigBuilder.setLogTopic(logTopic);
+ functionDetailsBuilder.setLogTopic(logTopic);
}
- functionConfigBuilder.setProcessingGuarantees(processingGuarantees);
+ functionDetailsBuilder.setProcessingGuarantees(processingGuarantees);
if (autoAck.equals("true")) {
- functionConfigBuilder.setAutoAck(true);
+ functionDetailsBuilder.setAutoAck(true);
} else {
- functionConfigBuilder.setAutoAck(false);
+ functionDetailsBuilder.setAutoAck(false);
}
if (userConfig != null && !userConfig.isEmpty()) {
Type type = new TypeToken<Map<String, String>>(){}.getType();
Map<String, String> userConfigMap = new Gson().fromJson(userConfig, type);
- functionConfigBuilder.putAllUserConfig(userConfigMap);
+ functionDetailsBuilder.putAllUserConfig(userConfigMap);
}
- FunctionConfig functionConfig = functionConfigBuilder.build();
- instanceConfig.setFunctionConfig(functionConfig);
+ FunctionDetails functionDetails = functionDetailsBuilder.build();
+ instanceConfig.setFunctionDetails(functionDetails);
ThreadRuntimeFactory containerFactory = new ThreadRuntimeFactory(
"LocalRunnerThreadGroup",
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
index 1bdedd7..0e8b300 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
@@ -72,17 +72,17 @@ class ProcessRuntime implements Runtime {
String codeFile,
String pulsarServiceUrl) {
List<String> args = new LinkedList<>();
- if (instanceConfig.getFunctionConfig().getRuntime() == Function.FunctionConfig.Runtime.JAVA) {
+ if (instanceConfig.getFunctionDetails().getRuntime() == Function.FunctionDetails.Runtime.JAVA) {
args.add("java");
args.add("-cp");
args.add(instanceFile);
args.add("-Dlog4j.configurationFile=java_instance_log4j2.yml");
args.add("-Dpulsar.log.dir=" + logDirectory);
- args.add("-Dpulsar.log.file=" + instanceConfig.getFunctionConfig().getName());
+ args.add("-Dpulsar.log.file=" + instanceConfig.getFunctionDetails().getName());
args.add("org.apache.pulsar.functions.runtime.JavaInstanceMain");
args.add("--jar");
args.add(codeFile);
- } else if (instanceConfig.getFunctionConfig().getRuntime() == Function.FunctionConfig.Runtime.PYTHON) {
+ } else if (instanceConfig.getFunctionDetails().getRuntime() == Function.FunctionDetails.Runtime.PYTHON) {
args.add("python");
args.add(instanceFile);
args.add("--py");
@@ -90,7 +90,7 @@ class ProcessRuntime implements Runtime {
args.add("--logging_directory");
args.add(logDirectory);
args.add("--logging_file");
- args.add(instanceConfig.getFunctionConfig().getName());
+ args.add(instanceConfig.getFunctionDetails().getName());
}
args.add("--instance_id");
args.add(instanceConfig.getInstanceId());
@@ -99,22 +99,22 @@ class ProcessRuntime implements Runtime {
args.add("--function_version");
args.add(instanceConfig.getFunctionVersion());
args.add("--tenant");
- args.add(instanceConfig.getFunctionConfig().getTenant());
+ args.add(instanceConfig.getFunctionDetails().getTenant());
args.add("--namespace");
- args.add(instanceConfig.getFunctionConfig().getNamespace());
+ args.add(instanceConfig.getFunctionDetails().getNamespace());
args.add("--name");
- args.add(instanceConfig.getFunctionConfig().getName());
+ args.add(instanceConfig.getFunctionDetails().getName());
args.add("--function_classname");
- args.add(instanceConfig.getFunctionConfig().getClassName());
- if (instanceConfig.getFunctionConfig().getLogTopic() != null &&
- !instanceConfig.getFunctionConfig().getLogTopic().isEmpty()) {
+ args.add(instanceConfig.getFunctionDetails().getClassName());
+ if (instanceConfig.getFunctionDetails().getLogTopic() != null &&
+ !instanceConfig.getFunctionDetails().getLogTopic().isEmpty()) {
args.add("--log_topic");
- args.add(instanceConfig.getFunctionConfig().getLogTopic());
+ args.add(instanceConfig.getFunctionDetails().getLogTopic());
}
- if (instanceConfig.getFunctionConfig().getCustomSerdeInputsCount() > 0) {
+ if (instanceConfig.getFunctionDetails().getCustomSerdeInputsCount() > 0) {
String inputTopicString = "";
String inputSerdeClassNameString = "";
- for (Map.Entry<String, String> entry : instanceConfig.getFunctionConfig().getCustomSerdeInputsMap().entrySet()) {
+ for (Map.Entry<String, String> entry : instanceConfig.getFunctionDetails().getCustomSerdeInputsMap().entrySet()) {
if (inputTopicString.isEmpty()) {
inputTopicString = entry.getKey();
} else {
@@ -131,9 +131,9 @@ class ProcessRuntime implements Runtime {
args.add("--custom_serde_classnames");
args.add(inputSerdeClassNameString);
}
- if (instanceConfig.getFunctionConfig().getInputsCount() > 0) {
+ if (instanceConfig.getFunctionDetails().getInputsCount() > 0) {
String inputTopicString = "";
- for (String topicName : instanceConfig.getFunctionConfig().getInputsList()) {
+ for (String topicName : instanceConfig.getFunctionDetails().getInputsList()) {
if (inputTopicString.isEmpty()) {
inputTopicString = topicName;
} else {
@@ -144,28 +144,28 @@ class ProcessRuntime implements Runtime {
args.add(inputTopicString);
}
args.add("--auto_ack");
- if (instanceConfig.getFunctionConfig().getAutoAck()) {
+ if (instanceConfig.getFunctionDetails().getAutoAck()) {
args.add("true");
} else {
args.add("false");
}
- if (instanceConfig.getFunctionConfig().getOutput() != null
- && !instanceConfig.getFunctionConfig().getOutput().isEmpty()) {
+ if (instanceConfig.getFunctionDetails().getOutput() != null
+ && !instanceConfig.getFunctionDetails().getOutput().isEmpty()) {
args.add("--output_topic");
- args.add(instanceConfig.getFunctionConfig().getOutput());
+ args.add(instanceConfig.getFunctionDetails().getOutput());
}
- if (instanceConfig.getFunctionConfig().getOutputSerdeClassName() != null
- && !instanceConfig.getFunctionConfig().getOutputSerdeClassName().isEmpty()) {
+ if (instanceConfig.getFunctionDetails().getOutputSerdeClassName() != null
+ && !instanceConfig.getFunctionDetails().getOutputSerdeClassName().isEmpty()) {
args.add("--output_serde_classname");
- args.add(instanceConfig.getFunctionConfig().getOutputSerdeClassName());
+ args.add(instanceConfig.getFunctionDetails().getOutputSerdeClassName());
}
args.add("--processing_guarantees");
- args.add(String.valueOf(instanceConfig.getFunctionConfig().getProcessingGuarantees()));
+ args.add(String.valueOf(instanceConfig.getFunctionDetails().getProcessingGuarantees()));
args.add("--pulsar_serviceurl");
args.add(pulsarServiceUrl);
args.add("--max_buffered_tuples");
args.add(String.valueOf(instanceConfig.getMaxBufferedTuples()));
- Map<String, String> userConfig = instanceConfig.getFunctionConfig().getUserConfigMap();
+ Map<String, String> userConfig = instanceConfig.getFunctionDetails().getUserConfigMap();
if (userConfig != null && !userConfig.isEmpty()) {
args.add("--user_config");
args.add(new Gson().toJson(userConfig));
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntimeFactory.java
index 31673d2..2223cb7 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntimeFactory.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntimeFactory.java
@@ -85,7 +85,7 @@ public class ProcessRuntimeFactory implements RuntimeFactory {
@Override
public ProcessRuntime createContainer(InstanceConfig instanceConfig, String codeFile) {
String instanceFile;
- switch (instanceConfig.getFunctionConfig().getRuntime()) {
+ switch (instanceConfig.getFunctionDetails().getRuntime()) {
case JAVA:
instanceFile = javaInstanceJarFile;
break;
@@ -93,7 +93,7 @@ public class ProcessRuntimeFactory implements RuntimeFactory {
instanceFile = pythonInstanceFile;
break;
default:
- throw new RuntimeException("Unsupported Runtime " + instanceConfig.getFunctionConfig().getRuntime());
+ throw new RuntimeException("Unsupported Runtime " + instanceConfig.getFunctionDetails().getRuntime());
}
return new ProcessRuntime(
instanceConfig,
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java
index 6bcb988..272ca6a 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java
@@ -57,7 +57,7 @@ public class RuntimeSpawner implements AutoCloseable {
}
public void start() throws Exception {
- log.info("RuntimeSpawner starting function {} - {}", this.instanceConfig.getFunctionConfig().getName(),
+ log.info("RuntimeSpawner starting function {} - {}", this.instanceConfig.getFunctionDetails().getName(),
this.instanceConfig.getInstanceId());
runtime = runtimeFactory.createContainer(this.instanceConfig, codeFile);
runtime.start();
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ThreadRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ThreadRuntime.java
index 1d53a70..8cd858a 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ThreadRuntime.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ThreadRuntime.java
@@ -30,7 +30,7 @@ import org.apache.pulsar.functions.proto.InstanceCommunication;
import org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatus;
import org.apache.pulsar.functions.utils.functioncache.FunctionCacheManager;
import org.apache.pulsar.functions.instance.JavaInstanceRunnable;
-import org.apache.pulsar.functions.utils.FunctionConfigUtils;
+import org.apache.pulsar.functions.utils.FunctionDetailsUtils;
/**
* A function container implemented using java thread.
@@ -54,7 +54,7 @@ class ThreadRuntime implements Runtime {
PulsarClient pulsarClient,
String stateStorageServiceUrl) {
this.instanceConfig = instanceConfig;
- if (instanceConfig.getFunctionConfig().getRuntime() != Function.FunctionConfig.Runtime.JAVA) {
+ if (instanceConfig.getFunctionDetails().getRuntime() != Function.FunctionDetails.Runtime.JAVA) {
throw new RuntimeException("Thread Container only supports Java Runtime");
}
this.javaInstanceRunnable = new JavaInstanceRunnable(
@@ -74,7 +74,7 @@ class ThreadRuntime implements Runtime {
log.info("ThreadContainer starting function with instance config {}", instanceConfig);
startupException = null;
this.fnThread = new Thread(threadGroup, javaInstanceRunnable,
- FunctionConfigUtils.getFullyQualifiedName(instanceConfig.getFunctionConfig()));
+ FunctionDetailsUtils.getFullyQualifiedName(instanceConfig.getFunctionDetails()));
this.fnThread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
@Override
public void uncaughtException(Thread t, Throwable e) {
diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java
index d19d57d..48602e3 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java
@@ -21,7 +21,7 @@ package org.apache.pulsar.functions.runtime;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.functions.instance.InstanceConfig;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails;
import org.apache.pulsar.functions.runtime.ProcessRuntime;
import org.apache.pulsar.functions.runtime.ProcessRuntimeFactory;
import org.apache.pulsar.functions.runtime.ThreadRuntime;
@@ -68,25 +68,25 @@ public class ProcessRuntimeTest {
this.factory.close();
}
- FunctionConfig createFunctionConfig(FunctionConfig.Runtime runtime) {
- FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder();
- functionConfigBuilder.setRuntime(runtime);
- functionConfigBuilder.setTenant(TEST_TENANT);
- functionConfigBuilder.setNamespace(TEST_NAMESPACE);
- functionConfigBuilder.setName(TEST_NAME);
- functionConfigBuilder.setClassName("org.apache.pulsar.functions.utils.functioncache.AddFunction");
- functionConfigBuilder.addInputs(TEST_NAME + "-input1");
- functionConfigBuilder.addInputs(TEST_NAME + "-input2");
- functionConfigBuilder.setOutput(TEST_NAME + "-output");
- functionConfigBuilder.setOutputSerdeClassName("org.apache.pulsar.functions.runtime.serde.Utf8Serializer");
- functionConfigBuilder.setLogTopic(TEST_NAME + "-log");
- return functionConfigBuilder.build();
+ FunctionDetails createFunctionDetails(FunctionDetails.Runtime runtime) {
+ FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
+ functionDetailsBuilder.setRuntime(runtime);
+ functionDetailsBuilder.setTenant(TEST_TENANT);
+ functionDetailsBuilder.setNamespace(TEST_NAMESPACE);
+ functionDetailsBuilder.setName(TEST_NAME);
+ functionDetailsBuilder.setClassName("org.apache.pulsar.functions.utils.functioncache.AddFunction");
+ functionDetailsBuilder.addInputs(TEST_NAME + "-input1");
+ functionDetailsBuilder.addInputs(TEST_NAME + "-input2");
+ functionDetailsBuilder.setOutput(TEST_NAME + "-output");
+ functionDetailsBuilder.setOutputSerdeClassName("org.apache.pulsar.functions.runtime.serde.Utf8Serializer");
+ functionDetailsBuilder.setLogTopic(TEST_NAME + "-log");
+ return functionDetailsBuilder.build();
}
- InstanceConfig createJavaInstanceConfig(FunctionConfig.Runtime runtime) {
+ InstanceConfig createJavaInstanceConfig(FunctionDetails.Runtime runtime) {
InstanceConfig config = new InstanceConfig();
- config.setFunctionConfig(createFunctionConfig(runtime));
+ config.setFunctionDetails(createFunctionDetails(runtime));
config.setFunctionId(java.util.UUID.randomUUID().toString());
config.setFunctionVersion("1.0");
config.setInstanceId(java.util.UUID.randomUUID().toString());
@@ -97,26 +97,26 @@ public class ProcessRuntimeTest {
@Test
public void testJavaConstructor() {
- InstanceConfig config = createJavaInstanceConfig(FunctionConfig.Runtime.JAVA);
+ InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA);
ProcessRuntime container = factory.createContainer(config, userJarFile);
List<String> args = container.getProcessArgs();
assertEquals(args.size(), 41);
args.remove(args.size() - 1);
String expectedArgs = "java -cp " + javaInstanceJarFile + " -Dlog4j.configurationFile=java_instance_log4j2.yml "
- + "-Dpulsar.log.dir=" + logDirectory + "/functions" + " -Dpulsar.log.file=" + config.getFunctionConfig().getName()
+ + "-Dpulsar.log.dir=" + logDirectory + "/functions" + " -Dpulsar.log.file=" + config.getFunctionDetails().getName()
+ " org.apache.pulsar.functions.runtime.JavaInstanceMain"
+ " --jar " + userJarFile + " --instance_id "
+ config.getInstanceId() + " --function_id " + config.getFunctionId()
- + " --function_version " + config.getFunctionVersion() + " --tenant " + config.getFunctionConfig().getTenant()
- + " --namespace " + config.getFunctionConfig().getNamespace()
- + " --name " + config.getFunctionConfig().getName()
- + " --function_classname " + config.getFunctionConfig().getClassName()
- + " --log_topic " + config.getFunctionConfig().getLogTopic()
+ + " --function_version " + config.getFunctionVersion() + " --tenant " + config.getFunctionDetails().getTenant()
+ + " --namespace " + config.getFunctionDetails().getNamespace()
+ + " --name " + config.getFunctionDetails().getName()
+ + " --function_classname " + config.getFunctionDetails().getClassName()
+ + " --log_topic " + config.getFunctionDetails().getLogTopic()
+ " --input_topics " + TEST_NAME + "-input1," + TEST_NAME + "-input2"
+ " --auto_ack false"
- + " --output_topic " + config.getFunctionConfig().getOutput()
- + " --output_serde_classname " + config.getFunctionConfig().getOutputSerdeClassName()
+ + " --output_topic " + config.getFunctionDetails().getOutput()
+ + " --output_serde_classname " + config.getFunctionDetails().getOutputSerdeClassName()
+ " --processing_guarantees ATLEAST_ONCE"
+ " --pulsar_serviceurl " + pulsarServiceUrl
+ " --max_buffered_tuples 1024 --port";
@@ -125,7 +125,7 @@ public class ProcessRuntimeTest {
@Test
public void testPythonConstructor() {
- InstanceConfig config = createJavaInstanceConfig(FunctionConfig.Runtime.PYTHON);
+ InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.PYTHON);
ProcessRuntime container = factory.createContainer(config, userJarFile);
List<String> args = container.getProcessArgs();
@@ -133,17 +133,17 @@ public class ProcessRuntimeTest {
args.remove(args.size() - 1);
String expectedArgs = "python " + pythonInstanceFile
+ " --py " + userJarFile + " --logging_directory "
- + logDirectory + "/functions" + " --logging_file " + config.getFunctionConfig().getName() + " --instance_id "
+ + logDirectory + "/functions" + " --logging_file " + config.getFunctionDetails().getName() + " --instance_id "
+ config.getInstanceId() + " --function_id " + config.getFunctionId()
- + " --function_version " + config.getFunctionVersion() + " --tenant " + config.getFunctionConfig().getTenant()
- + " --namespace " + config.getFunctionConfig().getNamespace()
- + " --name " + config.getFunctionConfig().getName()
- + " --function_classname " + config.getFunctionConfig().getClassName()
- + " --log_topic " + config.getFunctionConfig().getLogTopic()
+ + " --function_version " + config.getFunctionVersion() + " --tenant " + config.getFunctionDetails().getTenant()
+ + " --namespace " + config.getFunctionDetails().getNamespace()
+ + " --name " + config.getFunctionDetails().getName()
+ + " --function_classname " + config.getFunctionDetails().getClassName()
+ + " --log_topic " + config.getFunctionDetails().getLogTopic()
+ " --input_topics " + TEST_NAME + "-input1," + TEST_NAME + "-input2"
+ " --auto_ack false"
- + " --output_topic " + config.getFunctionConfig().getOutput()
- + " --output_serde_classname " + config.getFunctionConfig().getOutputSerdeClassName()
+ + " --output_topic " + config.getFunctionDetails().getOutput()
+ + " --output_serde_classname " + config.getFunctionDetails().getOutputSerdeClassName()
+ " --processing_guarantees ATLEAST_ONCE"
+ " --pulsar_serviceurl " + pulsarServiceUrl
+ " --max_buffered_tuples 1024 --port";
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java
new file mode 100644
index 0000000..fe293ea
--- /dev/null
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.functions.utils;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+
+@Getter
+@Setter
+@Data
+@EqualsAndHashCode
+@ToString
+public class FunctionConfig {
+
+ public enum ProcessingGuarantees {
+ ATLEAST_ONCE,
+ ATMOST_ONCE,
+ EFFECTIVELY_ONCE
+ }
+
+ public enum SubscriptionType {
+ SHARED,
+ EXCLUSIVE
+ }
+
+ public enum Runtime {
+ JAVA,
+ PYTHON
+ }
+
+ private String tenant;
+ private String namespace;
+ private String name;
+ private String className;
+
+ private Collection<String> inputs = new LinkedList<>();
+ private Map<String, String> customSerdeInputs = new HashMap<>();
+
+ private String output;
+ private String outputSerdeClassName;
+
+ private String logTopic;
+ private ProcessingGuarantees processingGuarantees;
+ private Map<String, String> userConfig = new HashMap<>();
+ private SubscriptionType subscriptionType;
+ private Runtime runtime;
+ private boolean autoAck;
+ private int parallelism;
+ private String fqfn;
+}
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionDetailsUtils.java
similarity index 52%
rename from pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
rename to pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionDetailsUtils.java
index c715b03..7706030 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionDetailsUtils.java
@@ -19,33 +19,12 @@
package org.apache.pulsar.functions.utils;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails;
-import java.io.File;
-import java.io.IOException;
+public class FunctionDetailsUtils {
-public class FunctionConfigUtils {
-
- public static String convertYamlToJson(File file) throws IOException {
-
- ObjectMapper yamlReader = new ObjectMapper(new YAMLFactory());
- Object obj = yamlReader.readValue(file, Object.class);
-
- ObjectMapper jsonWriter = new ObjectMapper();
- return jsonWriter.writeValueAsString(obj);
- }
-
- public static FunctionConfig.Builder loadConfig(File file) throws IOException {
- String json = convertYamlToJson(file);
- FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder();
- Utils.mergeJson(json, functionConfigBuilder);
- return functionConfigBuilder;
- }
-
- public static String getFullyQualifiedName(FunctionConfig functionConfig) {
- return getFullyQualifiedName(functionConfig.getTenant(), functionConfig.getNamespace(), functionConfig.getName());
+ public static String getFullyQualifiedName(FunctionDetails FunctionDetails) {
+ return getFullyQualifiedName(FunctionDetails.getTenant(), FunctionDetails.getNamespace(), FunctionDetails.getName());
}
public static String getFullyQualifiedName(String tenant, String namespace, String functionName) {
@@ -64,34 +43,34 @@ public class FunctionConfigUtils {
return fullyQualifiedName.split("/")[2];
}
- public static boolean areAllRequiredFieldsPresent(FunctionConfig functionConfig) {
- if (functionConfig.getTenant() == null || functionConfig.getNamespace() == null
- || functionConfig.getName() == null || functionConfig.getClassName() == null
- || (functionConfig.getInputsCount() <= 0 && functionConfig.getCustomSerdeInputsCount() <= 0)
- || functionConfig.getParallelism() <= 0) {
+ public static boolean areAllRequiredFieldsPresent(FunctionDetails FunctionDetails) {
+ if (FunctionDetails.getTenant() == null || FunctionDetails.getNamespace() == null
+ || FunctionDetails.getName() == null || FunctionDetails.getClassName() == null
+ || (FunctionDetails.getInputsCount() <= 0 && FunctionDetails.getCustomSerdeInputsCount() <= 0)
+ || FunctionDetails.getParallelism() <= 0) {
return false;
} else {
return true;
}
}
- public static String getDownloadFileName(FunctionConfig functionConfig) {
- String[] hierarchy = functionConfig.getClassName().split("\\.");
+ public static String getDownloadFileName(FunctionDetails FunctionDetails) {
+ String[] hierarchy = FunctionDetails.getClassName().split("\\.");
String fileName;
if (hierarchy.length <= 0) {
- fileName = functionConfig.getClassName();
+ fileName = FunctionDetails.getClassName();
} else if (hierarchy.length == 1) {
fileName = hierarchy[0];
} else {
fileName = hierarchy[hierarchy.length - 2];
}
- switch (functionConfig.getRuntime()) {
+ switch (FunctionDetails.getRuntime()) {
case JAVA:
return fileName + ".jar";
case PYTHON:
return fileName + ".py";
default:
- throw new RuntimeException("Unknown runtime " + functionConfig.getRuntime());
+ throw new RuntimeException("Unknown runtime " + FunctionDetails.getRuntime());
}
}
}
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
index 1eb3d28..702f314 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
@@ -33,7 +33,7 @@ import org.apache.pulsar.functions.proto.Function.FunctionMetaData;
import org.apache.pulsar.functions.runtime.RuntimeFactory;
import org.apache.pulsar.functions.instance.InstanceConfig;
import org.apache.pulsar.functions.runtime.RuntimeSpawner;
-import org.apache.pulsar.functions.utils.FunctionConfigUtils;
+import org.apache.pulsar.functions.utils.FunctionDetailsUtils;
import java.io.File;
import java.io.FileOutputStream;
@@ -105,7 +105,7 @@ public class FunctionActioner implements AutoCloseable {
Function.Instance instance = functionRuntimeInfo.getFunctionInstance();
FunctionMetaData functionMetaData = instance.getFunctionMetaData();
log.info("Starting function {} - {} ...",
- functionMetaData.getFunctionConfig().getName(), instance.getInstanceId());
+ functionMetaData.getFunctionDetails().getName(), instance.getInstanceId());
File pkgDir = new File(
workerConfig.getDownloadDirectory(),
getDownloadPackagePath(functionMetaData));
@@ -115,7 +115,7 @@ public class FunctionActioner implements AutoCloseable {
File pkgFile = new File(
pkgDir,
- new File(FunctionConfigUtils.getDownloadFileName(functionMetaData.getFunctionConfig())).getName());
+ new File(FunctionDetailsUtils.getDownloadFileName(functionMetaData.getFunctionDetails())).getName());
if (!pkgFile.exists()) {
// download only when the package file doesn't exist
@@ -155,7 +155,7 @@ public class FunctionActioner implements AutoCloseable {
}
InstanceConfig instanceConfig = new InstanceConfig();
- instanceConfig.setFunctionConfig(functionMetaData.getFunctionConfig());
+ instanceConfig.setFunctionDetails(functionMetaData.getFunctionDetails());
// TODO: set correct function id and version when features implemented
instanceConfig.setFunctionId(UUID.randomUUID().toString());
instanceConfig.setFunctionVersion(UUID.randomUUID().toString());
@@ -172,7 +172,7 @@ public class FunctionActioner implements AutoCloseable {
Function.Instance instance = functionRuntimeInfo.getFunctionInstance();
FunctionMetaData functionMetaData = instance.getFunctionMetaData();
log.info("Stopping function {} - {}...",
- functionMetaData.getFunctionConfig().getName(), instance.getInstanceId());
+ functionMetaData.getFunctionDetails().getName(), instance.getInstanceId());
if (functionRuntimeInfo.getRuntimeSpawner() != null) {
functionRuntimeInfo.getRuntimeSpawner().close();
functionRuntimeInfo.setRuntimeSpawner(null);
@@ -188,7 +188,7 @@ public class FunctionActioner implements AutoCloseable {
FileUtils.deleteDirectory(pkgDir);
} catch (IOException e) {
log.warn("Failed to delete package for function: {}",
- FunctionConfigUtils.getFullyQualifiedName(functionMetaData.getFunctionConfig()), e);
+ FunctionDetailsUtils.getFullyQualifiedName(functionMetaData.getFunctionDetails()), e);
}
}
}
@@ -196,9 +196,9 @@ public class FunctionActioner implements AutoCloseable {
private String getDownloadPackagePath(FunctionMetaData functionMetaData) {
return StringUtils.join(
new String[]{
- functionMetaData.getFunctionConfig().getTenant(),
- functionMetaData.getFunctionConfig().getNamespace(),
- functionMetaData.getFunctionConfig().getName(),
+ functionMetaData.getFunctionDetails().getTenant(),
+ functionMetaData.getFunctionDetails().getNamespace(),
+ functionMetaData.getFunctionDetails().getName(),
},
File.separatorChar);
}
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java
index 8a8f95e..b5f9751 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java
@@ -153,7 +153,7 @@ public class FunctionMetaDataManager implements AutoCloseable {
return ret;
}
for (FunctionMetaData functionMetaData : this.functionMetaDataMap.get(tenant).get(namespace).values()) {
- ret.add(functionMetaData.getFunctionConfig().getName());
+ ret.add(functionMetaData.getFunctionDetails().getName());
}
return ret;
}
@@ -178,19 +178,19 @@ public class FunctionMetaDataManager implements AutoCloseable {
long version = 0;
- String tenant = functionMetaData.getFunctionConfig().getTenant();
+ String tenant = functionMetaData.getFunctionDetails().getTenant();
if (!this.functionMetaDataMap.containsKey(tenant)) {
this.functionMetaDataMap.put(tenant, new ConcurrentHashMap<>());
}
Map<String, Map<String, FunctionMetaData>> namespaces = this.functionMetaDataMap.get(tenant);
- String namespace = functionMetaData.getFunctionConfig().getNamespace();
+ String namespace = functionMetaData.getFunctionDetails().getNamespace();
if (!namespaces.containsKey(namespace)) {
namespaces.put(namespace, new ConcurrentHashMap<>());
}
Map<String, FunctionMetaData> functionMetaDatas = namespaces.get(namespace);
- String functionName = functionMetaData.getFunctionConfig().getName();
+ String functionName = functionMetaData.getFunctionDetails().getName();
if (functionMetaDatas.containsKey(functionName)) {
version = functionMetaDatas.get(functionName).getVersion() + 1;
}
@@ -251,12 +251,12 @@ public class FunctionMetaDataManager implements AutoCloseable {
*/
private boolean containsFunctionMetaData(FunctionMetaData functionMetaData) {
- return containsFunctionMetaData(functionMetaData.getFunctionConfig());
+ return containsFunctionMetaData(functionMetaData.getFunctionDetails());
}
- private boolean containsFunctionMetaData(Function.FunctionConfig functionConfig) {
+ private boolean containsFunctionMetaData(Function.FunctionDetails functionDetails) {
return containsFunctionMetaData(
- functionConfig.getTenant(), functionConfig.getNamespace(), functionConfig.getName());
+ functionDetails.getTenant(), functionDetails.getNamespace(), functionDetails.getName());
}
private boolean containsFunctionMetaData(String tenant, String namespace, String functionName) {
@@ -274,9 +274,9 @@ public class FunctionMetaDataManager implements AutoCloseable {
synchronized void proccessDeregister(Request.ServiceRequest deregisterRequest) {
FunctionMetaData deregisterRequestFs = deregisterRequest.getFunctionMetaData();
- String functionName = deregisterRequestFs.getFunctionConfig().getName();
- String tenant = deregisterRequestFs.getFunctionConfig().getTenant();
- String namespace = deregisterRequestFs.getFunctionConfig().getNamespace();
+ String functionName = deregisterRequestFs.getFunctionDetails().getName();
+ String tenant = deregisterRequestFs.getFunctionDetails().getTenant();
+ String namespace = deregisterRequestFs.getFunctionDetails().getNamespace();
boolean needsScheduling = false;
@@ -363,25 +363,25 @@ public class FunctionMetaDataManager implements AutoCloseable {
private boolean isRequestOutdated(Request.ServiceRequest serviceRequest) {
FunctionMetaData requestFunctionMetaData = serviceRequest.getFunctionMetaData();
- Function.FunctionConfig functionConfig = requestFunctionMetaData.getFunctionConfig();
- FunctionMetaData currentFunctionMetaData = this.functionMetaDataMap.get(functionConfig.getTenant())
- .get(functionConfig.getNamespace()).get(functionConfig.getName());
+ Function.FunctionDetails functionDetails = requestFunctionMetaData.getFunctionDetails();
+ FunctionMetaData currentFunctionMetaData = this.functionMetaDataMap.get(functionDetails.getTenant())
+ .get(functionDetails.getNamespace()).get(functionDetails.getName());
return currentFunctionMetaData.getVersion() >= requestFunctionMetaData.getVersion();
}
@VisibleForTesting
void setFunctionMetaData(FunctionMetaData functionMetaData) {
- Function.FunctionConfig functionConfig = functionMetaData.getFunctionConfig();
- if (!this.functionMetaDataMap.containsKey(functionConfig.getTenant())) {
- this.functionMetaDataMap.put(functionConfig.getTenant(), new ConcurrentHashMap<>());
+ Function.FunctionDetails functionDetails = functionMetaData.getFunctionDetails();
+ if (!this.functionMetaDataMap.containsKey(functionDetails.getTenant())) {
+ this.functionMetaDataMap.put(functionDetails.getTenant(), new ConcurrentHashMap<>());
}
- if (!this.functionMetaDataMap.get(functionConfig.getTenant()).containsKey(functionConfig.getNamespace())) {
- this.functionMetaDataMap.get(functionConfig.getTenant())
- .put(functionConfig.getNamespace(), new ConcurrentHashMap<>());
+ if (!this.functionMetaDataMap.get(functionDetails.getTenant()).containsKey(functionDetails.getNamespace())) {
+ this.functionMetaDataMap.get(functionDetails.getTenant())
+ .put(functionDetails.getNamespace(), new ConcurrentHashMap<>());
}
- this.functionMetaDataMap.get(functionConfig.getTenant())
- .get(functionConfig.getNamespace()).put(functionConfig.getName(), functionMetaData);
+ this.functionMetaDataMap.get(functionDetails.getTenant())
+ .get(functionDetails.getNamespace()).put(functionDetails.getName(), functionMetaData);
}
@VisibleForTesting
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java
index a20c782..f7e793a 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java
@@ -181,13 +181,13 @@ public class FunctionRuntimeManager implements AutoCloseable{
.filter(
assignment ->
(tenant.equals(assignment.getInstance()
- .getFunctionMetaData().getFunctionConfig()
+ .getFunctionMetaData().getFunctionDetails()
.getTenant())
&& namespace.equals((assignment.getInstance()
- .getFunctionMetaData().getFunctionConfig()
+ .getFunctionMetaData().getFunctionDetails()
.getNamespace()))
&& functionName.equals(assignment.getInstance()
- .getFunctionMetaData().getFunctionConfig()
+ .getFunctionMetaData().getFunctionDetails()
.getName())))
.collect(Collectors.toList()));
}
@@ -316,9 +316,9 @@ public class FunctionRuntimeManager implements AutoCloseable{
for (Assignment assignment : assignments) {
InstanceCommunication.FunctionStatus functionStatus = this.getFunctionInstanceStatus(
- assignment.getInstance().getFunctionMetaData().getFunctionConfig().getTenant(),
- assignment.getInstance().getFunctionMetaData().getFunctionConfig().getNamespace(),
- assignment.getInstance().getFunctionMetaData().getFunctionConfig().getName(),
+ assignment.getInstance().getFunctionMetaData().getFunctionDetails().getTenant(),
+ assignment.getInstance().getFunctionMetaData().getFunctionDetails().getNamespace(),
+ assignment.getInstance().getFunctionMetaData().getFunctionDetails().getName(),
assignment.getInstance().getInstanceId());
functionStatusListBuilder.addFunctionStatusList(functionStatus);
@@ -468,9 +468,9 @@ public class FunctionRuntimeManager implements AutoCloseable{
private Assignment findAssignment(Assignment assignment) {
return findAssignment(
- assignment.getInstance().getFunctionMetaData().getFunctionConfig().getTenant(),
- assignment.getInstance().getFunctionMetaData().getFunctionConfig().getNamespace(),
- assignment.getInstance().getFunctionMetaData().getFunctionConfig().getName(),
+ assignment.getInstance().getFunctionMetaData().getFunctionDetails().getTenant(),
+ assignment.getInstance().getFunctionMetaData().getFunctionDetails().getNamespace(),
+ assignment.getInstance().getFunctionMetaData().getFunctionDetails().getName(),
assignment.getInstance().getInstanceId()
);
}
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionsStatsGenerator.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionsStatsGenerator.java
index c9fd539..0a829cb 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionsStatsGenerator.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionsStatsGenerator.java
@@ -56,11 +56,11 @@ public class FunctionsStatsGenerator {
InstanceCommunication.MetricsData.DataDigest dataDigest = metricsEntry.getValue();
String tenant = functionRuntimeInfo.getFunctionInstance()
- .getFunctionMetaData().getFunctionConfig().getTenant();
+ .getFunctionMetaData().getFunctionDetails().getTenant();
String namespace = functionRuntimeInfo.getFunctionInstance()
- .getFunctionMetaData().getFunctionConfig().getNamespace();
+ .getFunctionMetaData().getFunctionDetails().getNamespace();
String name = functionRuntimeInfo.getFunctionInstance()
- .getFunctionMetaData().getFunctionConfig().getName();
+ .getFunctionMetaData().getFunctionDetails().getName();
int instanceId = functionRuntimeInfo.getFunctionInstance().getInstanceId();
String qualifiedNamespace = String.format("%s/%s", tenant, namespace);
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/MembershipManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/MembershipManager.java
index 4942b34..4e5fb21 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/MembershipManager.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/MembershipManager.java
@@ -47,7 +47,7 @@ import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.common.policies.data.ConsumerStats;
import org.apache.pulsar.common.policies.data.PersistentTopicStats;
import org.apache.pulsar.functions.proto.Function;
-import org.apache.pulsar.functions.utils.FunctionConfigUtils;
+import org.apache.pulsar.functions.utils.FunctionDetailsUtils;
/**
* A simple implementation of leader election using a pulsar topic.
@@ -180,7 +180,7 @@ public class MembershipManager implements AutoCloseable, ConsumerEventListener {
List<Function.FunctionMetaData> functionMetaDataList = functionMetaDataManager.getAllFunctionMetaData();
Map<String, Function.FunctionMetaData> functionMetaDataMap = new HashMap<>();
for (Function.FunctionMetaData entry : functionMetaDataList) {
- functionMetaDataMap.put(FunctionConfigUtils.getFullyQualifiedName(entry.getFunctionConfig()), entry);
+ functionMetaDataMap.put(FunctionDetailsUtils.getFullyQualifiedName(entry.getFunctionDetails()), entry);
}
Map<String, Map<String, Function.Assignment>> currentAssignments = functionRuntimeManager.getCurrentAssignments();
Map<String, Function.Assignment> assignmentMap = new HashMap<>();
@@ -193,8 +193,8 @@ public class MembershipManager implements AutoCloseable, ConsumerEventListener {
Iterator<Map.Entry<Function.Instance, Long>> it = unsignedFunctionDurations.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<Function.Instance, Long> entry = it.next();
- String fullyQualifiedFunctionName = FunctionConfigUtils.getFullyQualifiedName(
- entry.getKey().getFunctionMetaData().getFunctionConfig());
+ String fullyQualifiedFunctionName = FunctionDetailsUtils.getFullyQualifiedName(
+ entry.getKey().getFunctionMetaData().getFunctionDetails());
String fullyQualifiedInstanceId = Utils.getFullyQualifiedInstanceId(entry.getKey());
//remove functions that don't exist anymore
if (!functionMetaDataMap.containsKey(fullyQualifiedFunctionName)) {
@@ -215,9 +215,9 @@ public class MembershipManager implements AutoCloseable, ConsumerEventListener {
// check for function instances that haven't been assigned
for (Function.FunctionMetaData functionMetaData : functionMetaDataList) {
Collection<Function.Assignment> assignments
- = FunctionRuntimeManager.findFunctionAssignments(functionMetaData.getFunctionConfig().getTenant(),
- functionMetaData.getFunctionConfig().getNamespace(),
- functionMetaData.getFunctionConfig().getName(),
+ = FunctionRuntimeManager.findFunctionAssignments(functionMetaData.getFunctionDetails().getTenant(),
+ functionMetaData.getFunctionDetails().getNamespace(),
+ functionMetaData.getFunctionDetails().getName(),
currentAssignments);
Set<Function.Instance> assignedInstances = assignments.stream()
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java
index fd61161..c6b72a5 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java
@@ -192,7 +192,7 @@ public class SchedulerManager implements AutoCloseable {
public static List<Function.Instance> computeInstances(FunctionMetaData functionMetaData) {
List<Function.Instance> functionInstances = new LinkedList<>();
- int instances = functionMetaData.getFunctionConfig().getParallelism();
+ int instances = functionMetaData.getFunctionDetails().getParallelism();
for (int i = 0; i < instances; i++) {
functionInstances.add(Function.Instance.newBuilder()
.setFunctionMetaData(functionMetaData)
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Utils.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Utils.java
index d5d716a..fba562f 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Utils.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Utils.java
@@ -186,9 +186,9 @@ public final class Utils {
public static String getFullyQualifiedInstanceId(Function.Instance instance) {
return getFullyQualifiedInstanceId(
- instance.getFunctionMetaData().getFunctionConfig().getTenant(),
- instance.getFunctionMetaData().getFunctionConfig().getNamespace(),
- instance.getFunctionMetaData().getFunctionConfig().getName(),
+ instance.getFunctionMetaData().getFunctionDetails().getTenant(),
+ instance.getFunctionMetaData().getFunctionDetails().getNamespace(),
+ instance.getFunctionMetaData().getFunctionDetails().getName(),
instance.getInstanceId());
}
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
index f6d6523..1df2871 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
@@ -47,7 +47,7 @@ import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.Reader;
import org.apache.pulsar.common.policies.data.ErrorData;
import org.apache.pulsar.functions.proto.Function;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails;
import org.apache.pulsar.functions.proto.Function.FunctionMetaData;
import org.apache.pulsar.functions.proto.Function.PackageLocationMetaData;
import org.apache.pulsar.functions.proto.InstanceCommunication;
@@ -87,12 +87,12 @@ public class FunctionsImpl {
final @PathParam("functionName") String functionName,
final @FormDataParam("data") InputStream uploadedInputStream,
final @FormDataParam("data") FormDataContentDisposition fileDetail,
- final @FormDataParam("functionConfig") String functionConfigJson) {
- FunctionConfig functionConfig;
+ final @FormDataParam("functionDetails") String functionDetailsJson) {
+ FunctionDetails functionDetails;
// validate parameters
try {
- functionConfig = validateUpdateRequestParams(tenant, namespace, functionName,
- uploadedInputStream, fileDetail, functionConfigJson);
+ functionDetails = validateUpdateRequestParams(tenant, namespace, functionName,
+ uploadedInputStream, fileDetail, functionDetailsJson);
} catch (IllegalArgumentException e) {
log.error("Invalid register function request @ /{}/{}/{}",
tenant, namespace, functionName, e);
@@ -112,7 +112,7 @@ public class FunctionsImpl {
// function state
FunctionMetaData.Builder functionMetaDataBuilder = FunctionMetaData.newBuilder()
- .setFunctionConfig(functionConfig)
+ .setFunctionDetails(functionDetails)
.setCreateTime(System.currentTimeMillis())
.setVersion(0);
@@ -136,13 +136,13 @@ public class FunctionsImpl {
final @PathParam("functionName") String functionName,
final @FormDataParam("data") InputStream uploadedInputStream,
final @FormDataParam("data") FormDataContentDisposition fileDetail,
- final @FormDataParam("functionConfig") String functionConfigJson) {
+ final @FormDataParam("functionDetails") String functionDetailsJson) {
- FunctionConfig functionConfig;
+ FunctionDetails functionDetails;
// validate parameters
try {
- functionConfig = validateUpdateRequestParams(tenant, namespace, functionName,
- uploadedInputStream, fileDetail, functionConfigJson);
+ functionDetails = validateUpdateRequestParams(tenant, namespace, functionName,
+ uploadedInputStream, fileDetail, functionDetailsJson);
} catch (IllegalArgumentException e) {
log.error("Invalid update function request @ /{}/{}/{}",
tenant, namespace, functionName, e);
@@ -161,7 +161,7 @@ public class FunctionsImpl {
// function state
FunctionMetaData.Builder functionMetaDataBuilder = FunctionMetaData.newBuilder()
- .setFunctionConfig(functionConfig)
+ .setFunctionDetails(functionDetails)
.setCreateTime(System.currentTimeMillis())
.setVersion(0);
@@ -262,8 +262,8 @@ public class FunctionsImpl {
}
FunctionMetaData functionMetaData = functionMetaDataManager.getFunctionMetaData(tenant, namespace, functionName);
- String functionConfigJson = org.apache.pulsar.functions.utils.Utils.printJson(functionMetaData.getFunctionConfig());
- return Response.status(Status.OK).entity(functionConfigJson).build();
+ String functionDetailsJson = org.apache.pulsar.functions.utils.Utils.printJson(functionMetaData.getFunctionDetails());
+ return Response.status(Status.OK).entity(functionDetailsJson).build();
}
@GET
@@ -302,8 +302,8 @@ public class FunctionsImpl {
log.error("Got Exception Getting Status", e);
FunctionStatus.Builder functionStatusBuilder = FunctionStatus.newBuilder();
functionStatusBuilder.setRunning(false);
- String functionConfigJson = org.apache.pulsar.functions.utils.Utils.printJson(functionStatusBuilder.build());
- return Response.status(Status.OK).entity(functionConfigJson).build();
+ String functionDetailsJson = org.apache.pulsar.functions.utils.Utils.printJson(functionStatusBuilder.build());
+ return Response.status(Status.OK).entity(functionDetailsJson).build();
}
String jsonResponse = org.apache.pulsar.functions.utils.Utils.printJson(functionStatus);
@@ -344,8 +344,8 @@ public class FunctionsImpl {
log.error("Got Exception Getting Status", e);
FunctionStatus.Builder functionStatusBuilder = FunctionStatus.newBuilder();
functionStatusBuilder.setRunning(false);
- String functionConfigJson = org.apache.pulsar.functions.utils.Utils.printJson(functionStatusBuilder.build());
- return Response.status(Status.OK).entity(functionConfigJson).build();
+ String functionDetailsJson = org.apache.pulsar.functions.utils.Utils.printJson(functionStatusBuilder.build());
+ return Response.status(Status.OK).entity(functionDetailsJson).build();
}
String jsonResponse = org.apache.pulsar.functions.utils.Utils.printJson(functionStatusList);
@@ -452,7 +452,7 @@ public class FunctionsImpl {
final @PathParam("name") String functionName,
final @FormDataParam("data") String input,
final @FormDataParam("dataStream") InputStream uploadedInputStream) {
- FunctionConfig functionConfig;
+ FunctionDetails functionDetails;
// validate parameters
try {
validateTriggerRequestParams(tenant, namespace, functionName, input, uploadedInputStream);
@@ -475,12 +475,12 @@ public class FunctionsImpl {
FunctionMetaData functionMetaData = functionMetaDataManager.getFunctionMetaData(tenant, namespace, functionName);
String inputTopicToWrite;
- if (functionMetaData.getFunctionConfig().getInputsList().size() > 0) {
- inputTopicToWrite = functionMetaData.getFunctionConfig().getInputsList().get(0);
+ if (functionMetaData.getFunctionDetails().getInputsList().size() > 0) {
+ inputTopicToWrite = functionMetaData.getFunctionDetails().getInputsList().get(0);
} else {
- inputTopicToWrite = functionMetaData.getFunctionConfig().getCustomSerdeInputs().entrySet().iterator().next().getKey();
+ inputTopicToWrite = functionMetaData.getFunctionDetails().getCustomSerdeInputs().entrySet().iterator().next().getKey();
}
- String outputTopic = functionMetaData.getFunctionConfig().getOutput();
+ String outputTopic = functionMetaData.getFunctionDetails().getOutput();
Reader reader = null;
Producer producer = null;
try {
@@ -577,12 +577,12 @@ public class FunctionsImpl {
}
}
- private FunctionConfig validateUpdateRequestParams(String tenant,
+ private FunctionDetails validateUpdateRequestParams(String tenant,
String namespace,
String functionName,
InputStream uploadedInputStream,
FormDataContentDisposition fileDetail,
- String functionConfigJson) throws IllegalArgumentException {
+ String functionDetailsJson) throws IllegalArgumentException {
if (tenant == null) {
throw new IllegalArgumentException("Tenant is not provided");
}
@@ -595,42 +595,42 @@ public class FunctionsImpl {
if (uploadedInputStream == null || fileDetail == null) {
throw new IllegalArgumentException("Function Package is not provided");
}
- if (functionConfigJson == null) {
- throw new IllegalArgumentException("FunctionConfig is not provided");
+ if (functionDetailsJson == null) {
+ throw new IllegalArgumentException("FunctionDetails is not provided");
}
try {
- FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder();
- org.apache.pulsar.functions.utils.Utils.mergeJson(functionConfigJson, functionConfigBuilder);
- FunctionConfig functionConfig = functionConfigBuilder.build();
+ FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
+ org.apache.pulsar.functions.utils.Utils.mergeJson(functionDetailsJson, functionDetailsBuilder);
+ FunctionDetails functionDetails = functionDetailsBuilder.build();
List<String> missingFields = new LinkedList<>();
- if (functionConfig.getTenant() == null || functionConfig.getTenant().isEmpty()) {
+ if (functionDetails.getTenant() == null || functionDetails.getTenant().isEmpty()) {
missingFields.add("Tenant");
}
- if (functionConfig.getNamespace() == null || functionConfig.getNamespace().isEmpty()) {
+ if (functionDetails.getNamespace() == null || functionDetails.getNamespace().isEmpty()) {
missingFields.add("Namespace");
}
- if (functionConfig.getName() == null || functionConfig.getName().isEmpty()) {
+ if (functionDetails.getName() == null || functionDetails.getName().isEmpty()) {
missingFields.add("Name");
}
- if (functionConfig.getClassName() == null || functionConfig.getClassName().isEmpty()) {
+ if (functionDetails.getClassName() == null || functionDetails.getClassName().isEmpty()) {
missingFields.add("ClassName");
}
- if (functionConfig.getInputsCount() == 0 && functionConfig.getCustomSerdeInputsCount() == 0) {
+ if (functionDetails.getInputsCount() == 0 && functionDetails.getCustomSerdeInputsCount() == 0) {
missingFields.add("Input");
}
if (!missingFields.isEmpty()) {
String errorMessage = StringUtils.join(missingFields, ",");
throw new IllegalArgumentException(errorMessage + " is not provided");
}
- if (functionConfig.getParallelism() <= 0) {
+ if (functionDetails.getParallelism() <= 0) {
throw new IllegalArgumentException("Parallelism needs to be set to a positive number");
}
- return functionConfig;
+ return functionDetails;
} catch (IllegalArgumentException ex) {
throw ex;
} catch (Exception ex) {
- throw new IllegalArgumentException("Invalid FunctionConfig");
+ throw new IllegalArgumentException("Invalid FunctionDetails");
}
}
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java
index 1d7a8b6..d2347ff 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java
@@ -47,10 +47,10 @@ public class FunctionApiV2Resource extends FunctionApiResource {
final @PathParam("functionName") String functionName,
final @FormDataParam("data") InputStream uploadedInputStream,
final @FormDataParam("data") FormDataContentDisposition fileDetail,
- final @FormDataParam("functionConfig") String functionConfigJson) {
+ final @FormDataParam("functionDetails") String functionDetailsJson) {
return functions.registerFunction(
- tenant, namespace, functionName, uploadedInputStream, fileDetail, functionConfigJson);
+ tenant, namespace, functionName, uploadedInputStream, fileDetail, functionDetailsJson);
}
@@ -62,10 +62,10 @@ public class FunctionApiV2Resource extends FunctionApiResource {
final @PathParam("functionName") String functionName,
final @FormDataParam("data") InputStream uploadedInputStream,
final @FormDataParam("data") FormDataContentDisposition fileDetail,
- final @FormDataParam("functionConfig") String functionConfigJson) {
+ final @FormDataParam("functionDetails") String functionDetailsJson) {
return functions.updateFunction(
- tenant, namespace, functionName, uploadedInputStream, fileDetail, functionConfigJson);
+ tenant, namespace, functionName, uploadedInputStream, fileDetail, functionDetailsJson);
}
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionMetaDataManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionMetaDataManagerTest.java
index eb37e77..3d27fb4 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionMetaDataManagerTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionMetaDataManagerTest.java
@@ -50,15 +50,15 @@ public class FunctionMetaDataManagerTest {
mock(PulsarClient.class)));
Map<String, Function.FunctionMetaData> functionMetaDataMap1 = new HashMap<>();
- functionMetaDataMap1.put("func-1", Function.FunctionMetaData.newBuilder().setFunctionConfig(
- Function.FunctionConfig.newBuilder().setName("func-1")).build());
+ functionMetaDataMap1.put("func-1", Function.FunctionMetaData.newBuilder().setFunctionDetails(
+ Function.FunctionDetails.newBuilder().setName("func-1")).build());
functionMetaDataMap1.put("func-2",
- Function.FunctionMetaData.newBuilder().setFunctionConfig(
- Function.FunctionConfig.newBuilder().setName("func-2")).build());
+ Function.FunctionMetaData.newBuilder().setFunctionDetails(
+ Function.FunctionDetails.newBuilder().setName("func-2")).build());
Map<String, Function.FunctionMetaData> functionMetaDataInfoMap2 = new HashMap<>();
functionMetaDataInfoMap2.put("func-3",
- Function.FunctionMetaData.newBuilder().setFunctionConfig(
- Function.FunctionConfig.newBuilder().setName("func-3")).build());
+ Function.FunctionMetaData.newBuilder().setFunctionDetails(
+ Function.FunctionDetails.newBuilder().setName("func-3")).build());
functionMetaDataManager.functionMetaDataMap.put("tenant-1", new HashMap<>());
@@ -89,7 +89,7 @@ public class FunctionMetaDataManagerTest {
mock(SchedulerManager.class),
mock(PulsarClient.class)));
Function.FunctionMetaData m1 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")).build();
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")).build();
Mockito.doReturn(null).when(functionMetaDataManager).submit(any(Request.ServiceRequest.class));
functionMetaDataManager.updateFunction(m1);
@@ -126,7 +126,7 @@ public class FunctionMetaDataManagerTest {
mock(PulsarClient.class)));
Map<String, Function.FunctionMetaData> functionMetaDataMap = new HashMap<>();
Function.FunctionMetaData m2 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build();
functionMetaDataMap.put("func-1", m2);
functionMetaDataManager.functionMetaDataMap.put("tenant-1", new HashMap<>());
@@ -145,7 +145,7 @@ public class FunctionMetaDataManagerTest {
Request.ServiceRequest.ServiceRequestType.UPDATE)) {
return false;
}
- if (!serviceRequest.getFunctionMetaData().getFunctionConfig().equals(m2.getFunctionConfig())) {
+ if (!serviceRequest.getFunctionMetaData().getFunctionDetails().equals(m2.getFunctionDetails())) {
return false;
}
if (serviceRequest.getFunctionMetaData().getVersion() != (version + 1)) {
@@ -169,7 +169,7 @@ public class FunctionMetaDataManagerTest {
mock(SchedulerManager.class),
mock(PulsarClient.class)));
Function.FunctionMetaData m1 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build();
Map<String, Function.FunctionMetaData> functionMetaDataMap = new HashMap<>();
functionMetaDataMap.put("func-1", m1);
@@ -190,7 +190,7 @@ public class FunctionMetaDataManagerTest {
Request.ServiceRequest.ServiceRequestType.DELETE)) {
return false;
}
- if (!serviceRequest.getFunctionMetaData().getFunctionConfig().equals(m1.getFunctionConfig())) {
+ if (!serviceRequest.getFunctionMetaData().getFunctionDetails().equals(m1.getFunctionDetails())) {
return false;
}
if (serviceRequest.getFunctionMetaData().getVersion() != (version + 1)) {
@@ -251,7 +251,7 @@ public class FunctionMetaDataManagerTest {
// worker has no record of function
Function.FunctionMetaData m1 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build();
Request.ServiceRequest serviceRequest = Request.ServiceRequest.newBuilder()
@@ -278,11 +278,11 @@ public class FunctionMetaDataManagerTest {
mock(PulsarClient.class)));
Function.FunctionMetaData m3 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build();
functionMetaDataManager.setFunctionMetaData(m3);
Function.FunctionMetaData outdated = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version - 1).build();
serviceRequest = Request.ServiceRequest.newBuilder()
@@ -299,7 +299,7 @@ public class FunctionMetaDataManagerTest {
verify(schedulerManager, times(0)).schedule();
Function.FunctionMetaData outdated2 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build();
serviceRequest = Request.ServiceRequest.newBuilder()
@@ -327,11 +327,11 @@ public class FunctionMetaDataManagerTest {
mock(PulsarClient.class)));
Function.FunctionMetaData m4 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build();
functionMetaDataManager.setFunctionMetaData(m4);
Function.FunctionMetaData m5 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version + 1).build();
serviceRequest = Request.ServiceRequest.newBuilder()
@@ -364,11 +364,11 @@ public class FunctionMetaDataManagerTest {
mock(PulsarClient.class)));
// worker has no record of function
Function.FunctionMetaData test = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2")
.setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build();
functionMetaDataManager.setFunctionMetaData(test);
Function.FunctionMetaData m1 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build();
Request.ServiceRequest serviceRequest = Request.ServiceRequest.newBuilder()
.setServiceRequestType(Request.ServiceRequest.ServiceRequestType.UPDATE)
@@ -391,7 +391,7 @@ public class FunctionMetaDataManagerTest {
mock(PulsarClient.class)));
functionMetaDataManager.setFunctionMetaData(test);
Function.FunctionMetaData m2 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build();
functionMetaDataManager.setFunctionMetaData(m2);
serviceRequest = Request.ServiceRequest.newBuilder()
@@ -419,12 +419,12 @@ public class FunctionMetaDataManagerTest {
functionMetaDataManager.setFunctionMetaData(test);
Function.FunctionMetaData m3 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version ).build();
functionMetaDataManager.setFunctionMetaData(m3);
Function.FunctionMetaData m4 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version +1).build();
serviceRequest = Request.ServiceRequest.newBuilder()
.setServiceRequestType(Request.ServiceRequest.ServiceRequestType.UPDATE)
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java
index 3bca55c..690f474 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java
@@ -55,7 +55,7 @@ public class FunctionRuntimeManagerTest {
}
@Override
- public void processRecord(InstanceCommunication.MetricsData record, Function.FunctionConfig functionConfig) {
+ public void processRecord(InstanceCommunication.MetricsData record, Function.FunctionDetails functionDetails) {
}
@@ -95,12 +95,12 @@ public class FunctionRuntimeManagerTest {
mock(MembershipManager.class)
));
- Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig(
- Function.FunctionConfig.newBuilder()
+ Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(
+ Function.FunctionDetails.newBuilder()
.setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build();
- Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionConfig(
- Function.FunctionConfig.newBuilder()
+ Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails(
+ Function.FunctionDetails.newBuilder()
.setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build();
Function.Assignment assignment1 = Function.Assignment.newBuilder()
@@ -188,12 +188,12 @@ public class FunctionRuntimeManagerTest {
mock(MembershipManager.class)
));
- Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig(
- Function.FunctionConfig.newBuilder()
+ Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(
+ Function.FunctionDetails.newBuilder()
.setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build();
- Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionConfig(
- Function.FunctionConfig.newBuilder()
+ Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails(
+ Function.FunctionDetails.newBuilder()
.setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build();
Function.Assignment assignment1 = Function.Assignment.newBuilder()
@@ -285,12 +285,12 @@ public class FunctionRuntimeManagerTest {
mock(MembershipManager.class)
));
- Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig(
- Function.FunctionConfig.newBuilder()
+ Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(
+ Function.FunctionDetails.newBuilder()
.setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build();
- Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionConfig(
- Function.FunctionConfig.newBuilder()
+ Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails(
+ Function.FunctionDetails.newBuilder()
.setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build();
Function.Assignment assignment1 = Function.Assignment.newBuilder()
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionStatsGeneratorTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionStatsGeneratorTest.java
index 5bdb812..7bd85e8 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionStatsGeneratorTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionStatsGeneratorTest.java
@@ -71,8 +71,8 @@ public class FunctionStatsGeneratorTest {
RuntimeSpawner runtimeSpawner = mock(RuntimeSpawner.class);
doReturn(runtime).when(runtimeSpawner).getRuntime();
- Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig(
- Function.FunctionConfig.newBuilder()
+ Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(
+ Function.FunctionDetails.newBuilder()
.setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build();
Function.Instance instance = Function.Instance.newBuilder()
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/MembershipManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/MembershipManagerTest.java
index 4fd72b9..14b56ac 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/MembershipManagerTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/MembershipManagerTest.java
@@ -46,7 +46,6 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
@@ -119,12 +118,12 @@ public class MembershipManagerTest {
Mockito.doReturn(workerInfoList).when(membershipManager).getCurrentMembership();
- Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig(
- Function.FunctionConfig.newBuilder()
+ Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(
+ Function.FunctionDetails.newBuilder()
.setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build();
- Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionConfig(
- Function.FunctionConfig.newBuilder()
+ Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails(
+ Function.FunctionDetails.newBuilder()
.setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build();
List<Function.FunctionMetaData> metaDataList = new LinkedList<>();
@@ -183,12 +182,12 @@ public class MembershipManagerTest {
Mockito.doReturn(workerInfoList).when(membershipManager).getCurrentMembership();
- Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig(
- Function.FunctionConfig.newBuilder()
+ Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(
+ Function.FunctionDetails.newBuilder()
.setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build();
- Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionConfig(
- Function.FunctionConfig.newBuilder()
+ Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails(
+ Function.FunctionDetails.newBuilder()
.setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build();
List<Function.FunctionMetaData> metaDataList = new LinkedList<>();
@@ -272,12 +271,12 @@ public class MembershipManagerTest {
Mockito.doReturn(workerInfoList).when(membershipManager).getCurrentMembership();
- Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig(
- Function.FunctionConfig.newBuilder().setParallelism(1)
+ Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(
+ Function.FunctionDetails.newBuilder().setParallelism(1)
.setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build();
- Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionConfig(
- Function.FunctionConfig.newBuilder().setParallelism(1)
+ Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails(
+ Function.FunctionDetails.newBuilder().setParallelism(1)
.setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build();
List<Function.FunctionMetaData> metaDataList = new LinkedList<>();
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/SchedulerManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/SchedulerManagerTest.java
index b115b17..64d6931 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/SchedulerManagerTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/SchedulerManagerTest.java
@@ -96,7 +96,7 @@ public class SchedulerManagerTest {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version)
.build();
functionMetaDataList.add(function1);
@@ -140,7 +140,7 @@ public class SchedulerManagerTest {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version)
.build();
functionMetaDataList.add(function1);
@@ -190,12 +190,12 @@ public class SchedulerManagerTest {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version)
.build();
Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2")
.setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version)
.build();
functionMetaDataList.add(function1);
@@ -253,13 +253,13 @@ public class SchedulerManagerTest {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version)
.build();
// simulate function2 got removed
Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2")
.setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version)
.build();
functionMetaDataList.add(function1);
@@ -319,12 +319,12 @@ public class SchedulerManagerTest {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version)
.build();
Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2")
.setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version)
.build();
functionMetaDataList.add(function1);
@@ -384,7 +384,7 @@ public class SchedulerManagerTest {
doReturn(producer).when(pulsarClient).createProducer(any(), any());
Function.FunctionMetaData function2Scaled = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2")
.setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version)
.build();
functionMetaDataList = new LinkedList<>();
@@ -428,12 +428,12 @@ public class SchedulerManagerTest {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version)
.build();
Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2")
.setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version)
.build();
functionMetaDataList.add(function1);
@@ -504,7 +504,7 @@ public class SchedulerManagerTest {
doReturn(producer).when(pulsarClient).createProducer(any(), any());
Function.FunctionMetaData function2Scaled = Function.FunctionMetaData.newBuilder()
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2")
.setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version)
.build();
functionMetaDataList = new LinkedList<>();
@@ -539,13 +539,13 @@ public class SchedulerManagerTest {
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
.setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/foo/bar1"))
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")
.setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version)
.build();
Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder()
.setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/foo/bar1"))
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2")
.setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version)
.build();
functionMetaDataList.add(function1);
@@ -617,7 +617,7 @@ public class SchedulerManagerTest {
Function.FunctionMetaData function2Updated = Function.FunctionMetaData.newBuilder()
.setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/foo/bar2"))
- .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2")
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2")
.setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version)
.build();
functionMetaDataList = new LinkedList<>();
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java
index 50c6b7b..5ae9e99 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java
@@ -47,7 +47,7 @@ import org.apache.pulsar.functions.api.Context;
import org.apache.pulsar.functions.api.Function;
import org.apache.pulsar.functions.api.utils.DefaultSerDe;
import org.apache.pulsar.functions.proto.Function.PackageLocationMetaData;
-import org.apache.pulsar.functions.proto.Function.FunctionConfig;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails;
import org.apache.pulsar.functions.proto.Function.FunctionMetaData;
import org.apache.pulsar.functions.worker.FunctionMetaDataManager;
import org.apache.pulsar.functions.worker.Utils;
@@ -295,40 +295,40 @@ public class FunctionApiV2ResourceTest {
Integer parallelism,
String missingFieldName
) throws IOException {
- FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder();
+ FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
if (tenant != null) {
- functionConfigBuilder.setTenant(tenant);
+ functionDetailsBuilder.setTenant(tenant);
}
if (namespace != null) {
- functionConfigBuilder.setNamespace(namespace);
+ functionDetailsBuilder.setNamespace(namespace);
}
if (function != null) {
- functionConfigBuilder.setName(function);
+ functionDetailsBuilder.setName(function);
}
if (outputTopic != null) {
- functionConfigBuilder.setOutput(outputTopic);
+ functionDetailsBuilder.setOutput(outputTopic);
}
if (inputTopic != null && inputSerdeClassName != null) {
- functionConfigBuilder.putCustomSerdeInputs(inputTopic, inputSerdeClassName);
+ functionDetailsBuilder.putCustomSerdeInputs(inputTopic, inputSerdeClassName);
}
if (outputSerdeClassName != null) {
- functionConfigBuilder.setOutputSerdeClassName(outputSerdeClassName);
+ functionDetailsBuilder.setOutputSerdeClassName(outputSerdeClassName);
}
if (className != null) {
- functionConfigBuilder.setClassName(className);
+ functionDetailsBuilder.setClassName(className);
}
if (parallelism != null) {
- functionConfigBuilder.setParallelism(parallelism);
+ functionDetailsBuilder.setParallelism(parallelism);
}
- FunctionConfig functionConfig = functionConfigBuilder.build();
+ FunctionDetails functionDetails = functionDetailsBuilder.build();
Response response = resource.registerFunction(
tenant,
namespace,
function,
inputStream,
details,
- org.apache.pulsar.functions.utils.Utils.printJson(functionConfig));
+ org.apache.pulsar.functions.utils.Utils.printJson(functionDetails));
assertEquals(Status.BAD_REQUEST.getStatusCode(), response.getStatus());
if (missingFieldName.equals("parallelism")) {
@@ -339,7 +339,7 @@ public class FunctionApiV2ResourceTest {
}
private Response registerDefaultFunction() throws IOException {
- FunctionConfig functionConfig = FunctionConfig.newBuilder()
+ FunctionDetails functionDetails = FunctionDetails.newBuilder()
.setTenant(tenant).setNamespace(namespace).setName(function)
.setOutput(outputTopic).putCustomSerdeInputs(inputTopic, inputSerdeClassName)
.setOutputSerdeClassName(outputSerdeClassName)
@@ -351,7 +351,7 @@ public class FunctionApiV2ResourceTest {
function,
mockedInputStream,
mockedFormData,
- org.apache.pulsar.functions.utils.Utils.printJson(functionConfig));
+ org.apache.pulsar.functions.utils.Utils.printJson(functionDetails));
}
@Test
@@ -615,40 +615,40 @@ public class FunctionApiV2ResourceTest {
Integer parallelism,
String missingFieldName
) throws IOException {
- FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder();
+ FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
if (tenant != null) {
- functionConfigBuilder.setTenant(tenant);
+ functionDetailsBuilder.setTenant(tenant);
}
if (namespace != null) {
- functionConfigBuilder.setNamespace(namespace);
+ functionDetailsBuilder.setNamespace(namespace);
}
if (function != null) {
- functionConfigBuilder.setName(function);
+ functionDetailsBuilder.setName(function);
}
if (outputTopic != null) {
- functionConfigBuilder.setOutput(outputTopic);
+ functionDetailsBuilder.setOutput(outputTopic);
}
if (inputTopic != null && inputSerdeClassName != null) {
- functionConfigBuilder.putCustomSerdeInputs(inputTopic, inputSerdeClassName);
+ functionDetailsBuilder.putCustomSerdeInputs(inputTopic, inputSerdeClassName);
}
if (outputSerdeClassName != null) {
- functionConfigBuilder.setOutputSerdeClassName(outputSerdeClassName);
+ functionDetailsBuilder.setOutputSerdeClassName(outputSerdeClassName);
}
if (className != null) {
- functionConfigBuilder.setClassName(className);
+ functionDetailsBuilder.setClassName(className);
}
if (parallelism != null) {
- functionConfigBuilder.setParallelism(parallelism);
+ functionDetailsBuilder.setParallelism(parallelism);
}
- FunctionConfig functionConfig = functionConfigBuilder.build();
+ FunctionDetails functionDetails = functionDetailsBuilder.build();
Response response = resource.updateFunction(
tenant,
namespace,
function,
inputStream,
details,
- org.apache.pulsar.functions.utils.Utils.printJson(functionConfig));
+ org.apache.pulsar.functions.utils.Utils.printJson(functionDetails));
assertEquals(Status.BAD_REQUEST.getStatusCode(), response.getStatus());
if (missingFieldName.equals("parallelism")) {
@@ -659,7 +659,7 @@ public class FunctionApiV2ResourceTest {
}
private Response updateDefaultFunction() throws IOException {
- FunctionConfig functionConfig = FunctionConfig.newBuilder()
+ FunctionDetails functionDetails = FunctionDetails.newBuilder()
.setTenant(tenant).setNamespace(namespace).setName(function)
.setOutput(outputTopic).putCustomSerdeInputs(inputTopic, inputSerdeClassName)
.setOutputSerdeClassName(outputSerdeClassName)
@@ -671,7 +671,7 @@ public class FunctionApiV2ResourceTest {
function,
mockedInputStream,
mockedFormData,
- org.apache.pulsar.functions.utils.Utils.printJson(functionConfig));
+ org.apache.pulsar.functions.utils.Utils.printJson(functionDetails));
}
@Test
@@ -933,19 +933,19 @@ public class FunctionApiV2ResourceTest {
public void testGetFunctionSuccess() throws Exception {
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
- FunctionConfig functionConfig = FunctionConfig.newBuilder()
+ FunctionDetails functionDetails = FunctionDetails.newBuilder()
.setClassName(className)
.putCustomSerdeInputs(inputTopic, inputSerdeClassName)
.setOutputSerdeClassName(outputSerdeClassName)
.setName(function)
.setNamespace(namespace)
- .setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATMOST_ONCE)
+ .setProcessingGuarantees(FunctionDetails.ProcessingGuarantees.ATMOST_ONCE)
.setOutput(outputTopic)
.setTenant(tenant)
.setParallelism(parallelism).build();
FunctionMetaData metaData = FunctionMetaData.newBuilder()
.setCreateTime(System.currentTimeMillis())
- .setFunctionConfig(functionConfig)
+ .setFunctionDetails(functionDetails)
.setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("/path/to/package"))
.setVersion(1234)
.build();
@@ -954,7 +954,7 @@ public class FunctionApiV2ResourceTest {
Response response = getDefaultFunctionInfo();
assertEquals(Status.OK.getStatusCode(), response.getStatus());
assertEquals(
- org.apache.pulsar.functions.utils.Utils.printJson(functionConfig),
+ org.apache.pulsar.functions.utils.Utils.printJson(functionDetails),
response.getEntity());
}
--
To stop receiving notification emails like this one, please contact
sijie@apache.org.