You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2024/02/26 19:36:30 UTC
(pulsar) 01/02: [improve][fn] Optimize Function Worker startup by lazy loading and direct zip/bytecode access (#22122)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-2.11
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 597cfa1488401c7682c588b6cb59b2ebf82fdff3
Author: Lari Hotari <lh...@apache.org>
AuthorDate: Mon Feb 26 16:47:48 2024 +0200
[improve][fn] Optimize Function Worker startup by lazy loading and direct zip/bytecode access (#22122)
(cherry picked from commit bbc62245c5ddba1de4b1e7cee4ab49334bc36277)
---
conf/functions_worker.yml | 15 +-
distribution/server/src/assemble/LICENSE.bin.txt | 4 +
pom.xml | 14 +
.../apache/pulsar/common/nar/NarClassLoader.java | 33 +-
.../org/apache/pulsar/common/nar/NarUnpacker.java | 27 +-
.../org/apache/pulsar/functions/LocalRunner.java | 283 +--
.../functions/runtime/JavaInstanceStarter.java | 19 +-
.../functions/runtime/thread/ThreadRuntime.java | 6 +-
.../pulsar/functions/worker/ConnectorsManager.java | 46 +-
.../pulsar/functions/worker/FunctionsManager.java | 44 +-
.../pulsar/functions/worker/WorkerConfig.java | 21 +-
pulsar-functions/utils/pom.xml | 11 +
.../pulsar/functions/utils/FunctionCommon.java | 352 +---
.../functions/utils/FunctionConfigUtils.java | 164 +-
.../functions/utils/FunctionFilePackage.java | 179 ++
.../functions/utils/FunctionRuntimeCommon.java | 170 ++
.../functions/utils/LoadedFunctionPackage.java | 89 +
.../pulsar/functions/utils/SinkConfigUtils.java | 110 +-
.../pulsar/functions/utils/SourceConfigUtils.java | 82 +-
.../utils/ValidatableFunctionPackage.java | 59 +
.../pulsar/functions/utils/ValidatorUtils.java | 207 +--
.../functions/utils/functions/FunctionArchive.java | 52 +-
.../functions/utils/functions/FunctionUtils.java | 74 +-
.../pulsar/functions/utils/io/Connector.java | 76 +-
.../pulsar/functions/utils/io/ConnectorUtils.java | 101 +-
.../pulsar/functions/utils/FunctionCommonTest.java | 92 +-
.../functions/utils/FunctionConfigUtilsTest.java | 57 +-
.../functions/utils/SinkConfigUtilsTest.java | 93 +-
.../pulsar/functions/worker/FunctionActioner.java | 13 +-
.../functions/worker/PulsarWorkerService.java | 8 +
.../functions/worker/rest/api/ComponentImpl.java | 91 +-
.../functions/worker/rest/api/FunctionsImpl.java | 63 +-
.../functions/worker/rest/api/SinksImpl.java | 132 +-
.../functions/worker/rest/api/SourcesImpl.java | 125 +-
.../worker/FunctionAssignmentTailerTest.java | 6 +-
.../worker/rest/api/FunctionsImplTest.java | 2 +-
.../rest/api/v2/FunctionApiV2ResourceTest.java | 1446 +--------------
.../AbstractFunctionApiResourceTest.java} | 816 ++++-----
.../rest/api/v3/AbstractFunctionsResourceTest.java | 323 ++++
.../rest/api/v3/FunctionApiV3ResourceTest.java | 1878 ++------------------
.../worker/rest/api/v3/SinkApiV3ResourceTest.java | 457 ++---
.../rest/api/v3/SourceApiV3ResourceTest.java | 407 ++---
.../conf/functions_worker.conf | 2 +-
.../integration/topologies/PulsarCluster.java | 39 +-
44 files changed, 2905 insertions(+), 5383 deletions(-)
diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml
index c4c44a8ed3f..08c782b8be5 100644
--- a/conf/functions_worker.yml
+++ b/conf/functions_worker.yml
@@ -43,6 +43,16 @@ metadataStoreOperationTimeoutSeconds: 30
# Metadata store cache expiry time in seconds
metadataStoreCacheExpirySeconds: 300
+# Specifies if the function worker should use classloading for validating submissions for built-in
+# connectors and functions. This is required for validateConnectorConfig to take effect.
+# Default is false.
+enableClassloadingOfBuiltinFiles: false
+
+# Specifies if the function worker should use classloading for validating submissions for external
+# connectors and functions. This is required for validateConnectorConfig to take effect.
+# Default is false.
+enableClassloadingOfExternalFiles: false
+
################################
# Function package management
################################
@@ -393,7 +403,10 @@ saslJaasServerRoleTokenSignerSecretPath:
connectorsDirectory: ./connectors
functionsDirectory: ./functions
-# Should connector config be validated during submission
+# Enables extended validation for connector config with fine-grain annotation based validation
+# during submission. Classloading with either enableClassloadingOfExternalFiles or
+# enableClassloadingOfBuiltinFiles must be enabled on the worker for this to take effect.
+# Default is false.
validateConnectorConfig: false
# Whether to initialize distributed log metadata by runtime.
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 85d08f2fd38..088d36e39fd 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -495,6 +495,10 @@ The Apache Software License, Version 2.0
* Jodah
- net.jodah-typetools-0.5.0.jar
- net.jodah-failsafe-2.4.4.jar
+ * Byte Buddy
+ - net.bytebuddy-byte-buddy-1.14.12.jar
+ * zt-zip
+ - org.zeroturnaround-zt-zip-1.17.jar
* Apache Avro
- org.apache.avro-avro-1.11.3.jar
- org.apache.avro-avro-protobuf-1.11.3.jar
diff --git a/pom.xml b/pom.xml
index c58d4ea90b2..f037c9ef0e5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -143,6 +143,8 @@ flexible messaging model and an intuitive client API.</description>
<puppycrawl.checkstyle.version>8.37</puppycrawl.checkstyle.version>
<dockerfile-maven.version>1.4.13</dockerfile-maven.version>
<typetools.version>0.5.0</typetools.version>
+ <byte-buddy.version>1.14.12</byte-buddy.version>
+ <zt-zip.version>1.17</zt-zip.version>
<protobuf3.version>3.19.6</protobuf3.version>
<protoc3.version>${protobuf3.version}</protoc3.version>
<grpc.version>1.55.3</grpc.version>
@@ -978,6 +980,18 @@ flexible messaging model and an intuitive client API.</description>
<version>${typetools.version}</version>
</dependency>
+ <dependency>
+ <groupId>net.bytebuddy</groupId>
+ <artifactId>byte-buddy</artifactId>
+ <version>${byte-buddy.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.zeroturnaround</groupId>
+ <artifactId>zt-zip</artifactId>
+ <version>${zt-zip.version}</version>
+ </dependency>
+
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-bom</artifactId>
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java
index c4864883788..0705e76bf58 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java
@@ -155,6 +155,11 @@ public class NarClassLoader extends URLClassLoader {
});
}
+ public static List<File> getClasspathFromArchive(File narPath, String narExtractionDirectory) throws IOException {
+ File unpacked = NarUnpacker.unpackNar(narPath, getNarExtractionDirectory(narExtractionDirectory));
+ return getClassPathEntries(unpacked);
+ }
+
private static File getNarExtractionDirectory(String configuredDirectory) {
return new File(configuredDirectory + "/" + TMP_DIR_PREFIX);
}
@@ -165,16 +170,11 @@ public class NarClassLoader extends URLClassLoader {
* @param narWorkingDirectory
* directory to explode nar contents to
* @param parent
- * @throws IllegalArgumentException
- * if the NAR is missing the Java Services API file for <tt>FlowFileProcessor</tt> implementations.
- * @throws ClassNotFoundException
- * if any of the <tt>FlowFileProcessor</tt> implementations defined by the Java Services API cannot be
- * loaded.
* @throws IOException
* if an error occurs while loading the NAR.
*/
private NarClassLoader(final File narWorkingDirectory, Set<String> additionalJars, ClassLoader parent)
- throws ClassNotFoundException, IOException {
+ throws IOException {
super(new URL[0], parent);
this.narWorkingDirectory = narWorkingDirectory;
@@ -239,22 +239,31 @@ public class NarClassLoader extends URLClassLoader {
* if the URL list could not be updated.
*/
private void updateClasspath(File root) throws IOException {
- addURL(root.toURI().toURL()); // for compiled classes, META-INF/, etc.
+ getClassPathEntries(root).forEach(f -> {
+ try {
+ addURL(f.toURI().toURL());
+ } catch (IOException e) {
+ log.error("Failed to add entry to classpath: {}", f, e);
+ }
+ });
+ }
+ static List<File> getClassPathEntries(File root) {
+ List<File> classPathEntries = new ArrayList<>();
+ classPathEntries.add(root);
File dependencies = new File(root, "META-INF/bundled-dependencies");
if (!dependencies.isDirectory()) {
- log.warn("{} does not contain META-INF/bundled-dependencies!", narWorkingDirectory);
+ log.warn("{} does not contain META-INF/bundled-dependencies!", root);
}
- addURL(dependencies.toURI().toURL());
+ classPathEntries.add(dependencies);
if (dependencies.isDirectory()) {
final File[] jarFiles = dependencies.listFiles(JAR_FILTER);
if (jarFiles != null) {
Arrays.sort(jarFiles, Comparator.comparing(File::getName));
- for (File libJar : jarFiles) {
- addURL(libJar.toURI().toURL());
- }
+ classPathEntries.addAll(Arrays.asList(jarFiles));
}
}
+ return classPathEntries;
}
@Override
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java
index b2c0a37a650..1554db5693f 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java
@@ -33,13 +33,14 @@ import java.io.InputStream;
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
+import java.nio.file.Path;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Base64;
import java.util.Enumeration;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
import lombok.extern.slf4j.Slf4j;
/**
@@ -114,16 +115,22 @@ public class NarUnpacker {
* if the NAR could not be unpacked.
*/
private static void unpack(final File nar, final File workingDirectory) throws IOException {
- try (JarFile jarFile = new JarFile(nar)) {
- Enumeration<JarEntry> jarEntries = jarFile.entries();
- while (jarEntries.hasMoreElements()) {
- JarEntry jarEntry = jarEntries.nextElement();
- String name = jarEntry.getName();
- File f = new File(workingDirectory, name);
- if (jarEntry.isDirectory()) {
+ Path workingDirectoryPath = workingDirectory.toPath().normalize();
+ try (ZipFile zipFile = new ZipFile(nar)) {
+ Enumeration<? extends ZipEntry> zipEntries = zipFile.entries();
+ while (zipEntries.hasMoreElements()) {
+ ZipEntry zipEntry = zipEntries.nextElement();
+ String name = zipEntry.getName();
+ Path targetFilePath = workingDirectoryPath.resolve(name).normalize();
+ if (!targetFilePath.startsWith(workingDirectoryPath)) {
+ log.error("Invalid zip file with entry '{}'", name);
+ throw new IOException("Invalid zip file. Aborting unpacking.");
+ }
+ File f = targetFilePath.toFile();
+ if (zipEntry.isDirectory()) {
FileUtils.ensureDirectoryExistAndCanReadAndWrite(f);
} else {
- makeFile(jarFile.getInputStream(jarEntry), f);
+ makeFile(zipFile.getInputStream(zipEntry), f);
}
}
}
diff --git a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java
index 2b9b25ed743..2b976278759 100644
--- a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java
+++ b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java
@@ -31,8 +31,9 @@ import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.InetSocketAddress;
+import java.net.URISyntaxException;
import java.nio.file.Files;
-import java.nio.file.Paths;
+import java.nio.file.Path;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -47,9 +48,12 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import lombok.Builder;
+import lombok.Value;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.common.functions.FunctionConfig;
+import org.apache.pulsar.common.functions.FunctionDefinition;
import org.apache.pulsar.common.functions.Utils;
+import org.apache.pulsar.common.io.ConnectorDefinition;
import org.apache.pulsar.common.io.SinkConfig;
import org.apache.pulsar.common.io.SourceConfig;
import org.apache.pulsar.common.nar.FileUtils;
@@ -59,6 +63,7 @@ import org.apache.pulsar.functions.instance.AuthenticationConfig;
import org.apache.pulsar.functions.instance.InstanceConfig;
import org.apache.pulsar.functions.instance.stats.FunctionCollectorRegistry;
import org.apache.pulsar.functions.proto.Function;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType;
import org.apache.pulsar.functions.runtime.RuntimeFactory;
import org.apache.pulsar.functions.runtime.RuntimeSpawner;
import org.apache.pulsar.functions.runtime.RuntimeUtils;
@@ -71,8 +76,11 @@ import org.apache.pulsar.functions.secretsproviderconfigurator.NameAndConfigBase
import org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator;
import org.apache.pulsar.functions.utils.FunctionCommon;
import org.apache.pulsar.functions.utils.FunctionConfigUtils;
+import org.apache.pulsar.functions.utils.FunctionRuntimeCommon;
+import org.apache.pulsar.functions.utils.LoadedFunctionPackage;
import org.apache.pulsar.functions.utils.SinkConfigUtils;
import org.apache.pulsar.functions.utils.SourceConfigUtils;
+import org.apache.pulsar.functions.utils.ValidatableFunctionPackage;
import org.apache.pulsar.functions.utils.functioncache.FunctionCacheEntry;
import org.apache.pulsar.functions.utils.functions.FunctionArchive;
import org.apache.pulsar.functions.utils.functions.FunctionUtils;
@@ -90,8 +98,7 @@ public class LocalRunner implements AutoCloseable {
private final String functionsDir;
private final Thread shutdownHook;
private final int instanceLivenessCheck;
- private ClassLoader userCodeClassLoader;
- private boolean userCodeClassLoaderCreated;
+ private UserCodeClassLoader userCodeClassLoader;
private RuntimeFactory runtimeFactory;
private HTTPServer metricsServer;
@@ -100,6 +107,12 @@ public class LocalRunner implements AutoCloseable {
PROCESS
}
+ @Value
+ private static class UserCodeClassLoader {
+ ClassLoader classLoader;
+ boolean classLoaderCreated;
+ }
+
public static class FunctionConfigConverter implements IStringConverter<FunctionConfig> {
@Override
public FunctionConfig convert(String value) {
@@ -257,11 +270,13 @@ public class LocalRunner implements AutoCloseable {
}
private static String getPulsarDirectory(String directory) {
- String pulsarHome = System.getenv("PULSAR_HOME");
- if (pulsarHome == null) {
- pulsarHome = Paths.get("").toAbsolutePath().toString();
+ final Path directoryPath;
+ if (System.getenv("PULSAR_HOME") != null) {
+ directoryPath = Path.of(System.getenv("PULSAR_HOME"), directory);
+ } else {
+ directoryPath = Path.of(directory);
}
- return Paths.get(pulsarHome, directory).toString();
+ return directoryPath.toAbsolutePath().toString();
}
private static File createNarExtractionTempDirectory() {
@@ -308,16 +323,19 @@ public class LocalRunner implements AutoCloseable {
runtimeFactory = null;
}
- if (userCodeClassLoaderCreated) {
- if (userCodeClassLoader instanceof Closeable) {
- try {
- ((Closeable) userCodeClassLoader).close();
- } catch (IOException e) {
- log.warn("Error closing classloader", e);
- }
+ closeClassLoaderIfneeded(userCodeClassLoader);
+ userCodeClassLoader = null;
+ }
+ }
+
+ private static void closeClassLoaderIfneeded(UserCodeClassLoader userCodeClassLoader) {
+ if (userCodeClassLoader != null && userCodeClassLoader.isClassLoaderCreated()) {
+ if (userCodeClassLoader.getClassLoader() instanceof Closeable) {
+ try {
+ ((Closeable) userCodeClassLoader.getClassLoader()).close();
+ } catch (IOException e) {
+ log.warn("Error closing classloader", e);
}
- userCodeClassLoaderCreated = false;
- userCodeClassLoader = null;
}
}
}
@@ -337,47 +355,14 @@ public class LocalRunner implements AutoCloseable {
parallelism = functionConfig.getParallelism();
if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
userCodeFile = functionConfig.getJar();
- ClassLoader builtInFunctionClassLoader = userCodeFile != null
- ? isBuiltInFunction(userCodeFile)
- : null;
- if (builtInFunctionClassLoader != null) {
- userCodeClassLoader = builtInFunctionClassLoader;
- functionDetails = FunctionConfigUtils.convert(
- functionConfig,
- FunctionConfigUtils.validateJavaFunction(functionConfig, builtInFunctionClassLoader));
- } else if (userCodeFile != null && Utils.isFunctionPackageUrlSupported(userCodeFile)) {
- File file = FunctionCommon.extractFileFromPkgURL(userCodeFile);
- ClassLoader functionClassLoader = FunctionCommon.getClassLoaderFromPackage(
- Function.FunctionDetails.ComponentType.FUNCTION,
- functionConfig.getClassName(), file, narExtractionDirectory);
- functionDetails = FunctionConfigUtils.convert(
- functionConfig,
- FunctionConfigUtils.validateJavaFunction(functionConfig, functionClassLoader));
- userCodeClassLoader = functionClassLoader;
- userCodeClassLoaderCreated = true;
- } else if (userCodeFile != null) {
- File file = new File(userCodeFile);
- if (!file.exists()) {
- throw new RuntimeException("User jar does not exist");
- }
- ClassLoader functionClassLoader = FunctionCommon.getClassLoaderFromPackage(
- Function.FunctionDetails.ComponentType.FUNCTION,
- functionConfig.getClassName(), file, narExtractionDirectory);
- functionDetails = FunctionConfigUtils.convert(
- functionConfig,
- FunctionConfigUtils.validateJavaFunction(functionConfig, functionClassLoader));
- userCodeClassLoader = functionClassLoader;
- userCodeClassLoaderCreated = true;
- } else {
- if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
- throw new IllegalStateException("The jar property must be specified in FunctionConfig.");
- }
- functionDetails = FunctionConfigUtils.convert(
- functionConfig,
- FunctionConfigUtils.validateJavaFunction(
- functionConfig,
- Thread.currentThread().getContextClassLoader()));
- }
+ userCodeClassLoader = extractClassLoader(
+ userCodeFile, ComponentType.FUNCTION, functionConfig.getClassName());
+ ValidatableFunctionPackage validatableFunctionPackage =
+ new LoadedFunctionPackage(getCurrentOrUserCodeClassLoader(),
+ FunctionDefinition.class);
+ functionDetails = FunctionConfigUtils.convert(
+ functionConfig,
+ FunctionConfigUtils.validateJavaFunction(functionConfig, validatableFunctionPackage));
} else if (functionConfig.getRuntime() == FunctionConfig.Runtime.GO) {
userCodeFile = functionConfig.getGo();
} else if (functionConfig.getRuntime() == FunctionConfig.Runtime.PYTHON) {
@@ -387,91 +372,34 @@ public class LocalRunner implements AutoCloseable {
}
if (functionDetails == null) {
- functionDetails = FunctionConfigUtils.convert(functionConfig,
- userCodeClassLoader != null ? userCodeClassLoader :
- Thread.currentThread().getContextClassLoader());
+ ValidatableFunctionPackage validatableFunctionPackage =
+ new LoadedFunctionPackage(getCurrentOrUserCodeClassLoader(),
+ FunctionDefinition.class);
+ functionDetails = FunctionConfigUtils.convert(functionConfig, validatableFunctionPackage);
}
} else if (sourceConfig != null) {
inferMissingArguments(sourceConfig);
userCodeFile = sourceConfig.getArchive();
parallelism = sourceConfig.getParallelism();
-
- ClassLoader builtInSourceClassLoader = userCodeFile != null ? isBuiltInSource(userCodeFile) : null;
- if (builtInSourceClassLoader != null) {
- functionDetails = SourceConfigUtils.convert(
- sourceConfig, SourceConfigUtils.validateAndExtractDetails(
- sourceConfig, builtInSourceClassLoader, true));
- userCodeClassLoader = builtInSourceClassLoader;
- } else if (userCodeFile != null && Utils.isFunctionPackageUrlSupported(userCodeFile)) {
- File file = FunctionCommon.extractFileFromPkgURL(userCodeFile);
- ClassLoader sourceClassLoader = FunctionCommon.getClassLoaderFromPackage(
- Function.FunctionDetails.ComponentType.SOURCE,
- sourceConfig.getClassName(), file, narExtractionDirectory);
- functionDetails = SourceConfigUtils.convert(
- sourceConfig,
- SourceConfigUtils.validateAndExtractDetails(sourceConfig, sourceClassLoader, true));
- userCodeClassLoader = sourceClassLoader;
- userCodeClassLoaderCreated = true;
- } else if (userCodeFile != null) {
- File file = new File(userCodeFile);
- if (!file.exists()) {
- throw new RuntimeException("Source archive (" + userCodeFile + ") does not exist");
- }
- ClassLoader sourceClassLoader = FunctionCommon.getClassLoaderFromPackage(
- Function.FunctionDetails.ComponentType.SOURCE,
- sourceConfig.getClassName(), file, narExtractionDirectory);
- functionDetails = SourceConfigUtils.convert(sourceConfig,
- SourceConfigUtils.validateAndExtractDetails(sourceConfig, sourceClassLoader, true));
- userCodeClassLoader = sourceClassLoader;
- userCodeClassLoaderCreated = true;
- } else {
- if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
- throw new IllegalStateException("The archive property must be specified in SourceConfig.");
- }
- functionDetails = SourceConfigUtils.convert(
- sourceConfig, SourceConfigUtils.validateAndExtractDetails(
- sourceConfig, Thread.currentThread().getContextClassLoader(), true));
- }
+ userCodeClassLoader = extractClassLoader(
+ userCodeFile, ComponentType.SOURCE, sourceConfig.getClassName());
+ ValidatableFunctionPackage validatableFunctionPackage =
+ new LoadedFunctionPackage(getCurrentOrUserCodeClassLoader(), ConnectorDefinition.class);
+ functionDetails = SourceConfigUtils.convert(sourceConfig,
+ SourceConfigUtils.validateAndExtractDetails(sourceConfig, validatableFunctionPackage, true));
} else if (sinkConfig != null) {
inferMissingArguments(sinkConfig);
userCodeFile = sinkConfig.getArchive();
parallelism = sinkConfig.getParallelism();
-
- ClassLoader builtInSinkClassLoader = userCodeFile != null ? isBuiltInSink(userCodeFile) : null;
- if (builtInSinkClassLoader != null) {
- functionDetails = SinkConfigUtils.convert(
- sinkConfig, SinkConfigUtils.validateAndExtractDetails(
- sinkConfig, builtInSinkClassLoader, true));
- userCodeClassLoader = builtInSinkClassLoader;
- } else if (Utils.isFunctionPackageUrlSupported(userCodeFile)) {
- File file = FunctionCommon.extractFileFromPkgURL(userCodeFile);
- ClassLoader sinkClassLoader = FunctionCommon.getClassLoaderFromPackage(
- Function.FunctionDetails.ComponentType.SINK,
- sinkConfig.getClassName(), file, narExtractionDirectory);
- functionDetails = SinkConfigUtils.convert(
- sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, true));
- userCodeClassLoader = sinkClassLoader;
- userCodeClassLoaderCreated = true;
- } else if (userCodeFile != null) {
- File file = new File(userCodeFile);
- if (!file.exists()) {
- throw new RuntimeException("Sink archive does not exist");
- }
- ClassLoader sinkClassLoader = FunctionCommon.getClassLoaderFromPackage(
- Function.FunctionDetails.ComponentType.SINK,
- sinkConfig.getClassName(), file, narExtractionDirectory);
- functionDetails = SinkConfigUtils.convert(
- sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, true));
- userCodeClassLoader = sinkClassLoader;
- userCodeClassLoaderCreated = true;
- } else {
- if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
- throw new IllegalStateException("The archive property must be specified in SourceConfig.");
- }
- functionDetails = SinkConfigUtils.convert(
- sinkConfig, SinkConfigUtils.validateAndExtractDetails(
- sinkConfig, Thread.currentThread().getContextClassLoader(), true));
- }
+ userCodeClassLoader = extractClassLoader(
+ userCodeFile, ComponentType.SINK, sinkConfig.getClassName());
+ ValidatableFunctionPackage validatableFunctionPackage =
+ new LoadedFunctionPackage(getCurrentOrUserCodeClassLoader(), ConnectorDefinition.class);
+
+ functionDetails = SinkConfigUtils.convert(
+ sinkConfig,
+ SinkConfigUtils.validateAndExtractDetails(sinkConfig, validatableFunctionPackage,
+ true));
} else {
throw new IllegalArgumentException("Must specify Function, Source or Sink config");
}
@@ -526,6 +454,67 @@ public class LocalRunner implements AutoCloseable {
}
}
+ private ClassLoader getCurrentOrUserCodeClassLoader() {
+ return userCodeClassLoader == null || userCodeClassLoader.getClassLoader() == null
+ ? Thread.currentThread().getContextClassLoader()
+ : userCodeClassLoader.getClassLoader();
+ }
+
+ private UserCodeClassLoader extractClassLoader(String userCodeFile, ComponentType componentType, String className)
+ throws IOException, URISyntaxException {
+ ClassLoader classLoader = userCodeFile != null ? isBuiltIn(userCodeFile, componentType) : null;
+ boolean classLoaderCreated = false;
+ if (classLoader == null) {
+ if (userCodeFile != null && Utils.isFunctionPackageUrlSupported(userCodeFile)) {
+ File file = FunctionCommon.extractFileFromPkgURL(userCodeFile);
+ classLoader = FunctionRuntimeCommon.getClassLoaderFromPackage(
+ componentType, className, file, narExtractionDirectory);
+ classLoaderCreated = true;
+ } else if (userCodeFile != null) {
+ File file = new File(userCodeFile);
+ if (!file.exists()) {
+ String errorMsg;
+ switch (componentType) {
+ case FUNCTION:
+ errorMsg = "User jar";
+ break;
+ case SOURCE:
+ errorMsg = "Source archive";
+ break;
+ case SINK:
+ errorMsg = "Sink archive";
+ break;
+ default:
+ throw new IllegalStateException("Unexpected value: " + componentType);
+ }
+ throw new RuntimeException(errorMsg + " (" + userCodeFile + ") does not exist");
+ }
+ classLoader = FunctionRuntimeCommon.getClassLoaderFromPackage(
+ componentType, className, file, narExtractionDirectory);
+ classLoaderCreated = true;
+ } else {
+ if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
+ String errorMsg;
+ switch (componentType) {
+ case FUNCTION:
+ errorMsg = "The jar property must be specified in FunctionConfig.";
+ break;
+ case SOURCE:
+ errorMsg = "The archive property must be specified in SourceConfig.";
+ break;
+ case SINK:
+ errorMsg = "The archive property must be specified in SinkConfig.";
+ break;
+ default:
+ throw new IllegalStateException("Unexpected ComponentType: " + componentType);
+ }
+ throw new IllegalStateException(errorMsg);
+ }
+ }
+ }
+ return new UserCodeClassLoader(classLoader, classLoaderCreated);
+ }
+
private void startProcessMode(org.apache.pulsar.functions.proto.Function.FunctionDetails functionDetails,
int parallelism, int instanceIdOffset, String serviceUrl,
String stateStorageServiceUrl, AuthenticationConfig authConfig,
@@ -641,8 +630,8 @@ public class LocalRunner implements AutoCloseable {
ClassLoader originalClassLoader = Thread.currentThread().getContextClassLoader();
try {
- if (userCodeClassLoader != null) {
- Thread.currentThread().setContextClassLoader(userCodeClassLoader);
+ if (userCodeClassLoader != null && userCodeClassLoader.getClassLoader() != null) {
+ Thread.currentThread().setContextClassLoader(userCodeClassLoader.getClassLoader());
}
runtimeFactory = new ThreadRuntimeFactory("LocalRunnerThreadGroup",
serviceUrl,
@@ -692,6 +681,20 @@ public class LocalRunner implements AutoCloseable {
}
}
+ private ClassLoader isBuiltIn(String component, ComponentType componentType)
+ throws IOException {
+ switch (componentType) {
+ case FUNCTION:
+ return isBuiltInFunction(component);
+ case SOURCE:
+ return isBuiltInSource(component);
+ case SINK:
+ return isBuiltInSink(component);
+ default:
+ throw new IllegalStateException("Unexpected ComponentType: " + componentType);
+ }
+ }
+
private ClassLoader isBuiltInFunction(String functionType) throws IOException {
// Validate the connector type from the locally available connectors
TreeMap<String, FunctionArchive> functions = getFunctions();
@@ -700,7 +703,7 @@ public class LocalRunner implements AutoCloseable {
FunctionArchive function = functions.get(functionName);
if (function != null && function.getFunctionDefinition().getFunctionClass() != null) {
// Function type is a valid built-in type.
- return function.getClassLoader();
+ return function.getFunctionPackage().getClassLoader();
} else {
return null;
}
@@ -714,7 +717,7 @@ public class LocalRunner implements AutoCloseable {
Connector connector = connectors.get(source);
if (connector != null && connector.getConnectorDefinition().getSourceClass() != null) {
// Source type is a valid built-in connector type.
- return connector.getClassLoader();
+ return connector.getConnectorFunctionPackage().getClassLoader();
} else {
return null;
}
@@ -728,18 +731,18 @@ public class LocalRunner implements AutoCloseable {
Connector connector = connectors.get(sink);
if (connector != null && connector.getConnectorDefinition().getSinkClass() != null) {
// Sink type is a valid built-in connector type
- return connector.getClassLoader();
+ return connector.getConnectorFunctionPackage().getClassLoader();
} else {
return null;
}
}
private TreeMap<String, FunctionArchive> getFunctions() throws IOException {
- return FunctionUtils.searchForFunctions(functionsDir);
+ return FunctionUtils.searchForFunctions(functionsDir, narExtractionDirectory, true);
}
private TreeMap<String, Connector> getConnectors() throws IOException {
- return ConnectorUtils.searchForConnectors(connectorsDir, narExtractionDirectory);
+ return ConnectorUtils.searchForConnectors(connectorsDir, narExtractionDirectory, true);
}
private SecretsProviderConfigurator getSecretsProviderConfigurator() {
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
index 5743341483f..f3ec46dcf28 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java
@@ -38,6 +38,9 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
+import net.bytebuddy.description.type.TypeDefinition;
+import net.bytebuddy.dynamic.ClassFileLocator;
+import net.bytebuddy.pool.TypePool;
import org.apache.commons.lang3.StringUtils;
import org.apache.pulsar.common.functions.WindowConfig;
import org.apache.pulsar.common.nar.NarClassLoader;
@@ -306,7 +309,8 @@ public class JavaInstanceStarter implements AutoCloseable {
}
private void inferringMissingTypeClassName(Function.FunctionDetails.Builder functionDetailsBuilder,
- ClassLoader classLoader) throws ClassNotFoundException {
+ ClassLoader classLoader) {
+ TypePool typePool = TypePool.Default.of(ClassFileLocator.ForClassLoader.of(classLoader));
switch (functionDetailsBuilder.getComponentType()) {
case FUNCTION:
if ((functionDetailsBuilder.hasSource()
@@ -325,14 +329,13 @@ public class JavaInstanceStarter implements AutoCloseable {
WindowConfig.class);
className = windowConfig.getActualWindowFunctionClassName();
}
-
- Class<?>[] typeArgs = FunctionCommon.getFunctionTypes(classLoader.loadClass(className),
+ TypeDefinition[] typeArgs = FunctionCommon.getFunctionTypes(typePool.describe(className).resolve(),
isWindowConfigPresent);
if (functionDetailsBuilder.hasSource()
&& functionDetailsBuilder.getSource().getTypeClassName().isEmpty()
&& typeArgs[0] != null) {
Function.SourceSpec.Builder sourceBuilder = functionDetailsBuilder.getSource().toBuilder();
- sourceBuilder.setTypeClassName(typeArgs[0].getName());
+ sourceBuilder.setTypeClassName(typeArgs[0].asErasure().getTypeName());
functionDetailsBuilder.setSource(sourceBuilder.build());
}
@@ -340,7 +343,7 @@ public class JavaInstanceStarter implements AutoCloseable {
&& functionDetailsBuilder.getSink().getTypeClassName().isEmpty()
&& typeArgs[1] != null) {
Function.SinkSpec.Builder sinkBuilder = functionDetailsBuilder.getSink().toBuilder();
- sinkBuilder.setTypeClassName(typeArgs[1].getName());
+ sinkBuilder.setTypeClassName(typeArgs[1].asErasure().getTypeName());
functionDetailsBuilder.setSink(sinkBuilder.build());
}
}
@@ -349,7 +352,8 @@ public class JavaInstanceStarter implements AutoCloseable {
if ((functionDetailsBuilder.hasSink()
&& functionDetailsBuilder.getSink().getTypeClassName().isEmpty())) {
String typeArg =
- getSinkType(functionDetailsBuilder.getSink().getClassName(), classLoader).getName();
+ getSinkType(functionDetailsBuilder.getSink().getClassName(), typePool).asErasure()
+ .getTypeName();
Function.SinkSpec.Builder sinkBuilder =
Function.SinkSpec.newBuilder(functionDetailsBuilder.getSink());
@@ -368,7 +372,8 @@ public class JavaInstanceStarter implements AutoCloseable {
if ((functionDetailsBuilder.hasSource()
&& functionDetailsBuilder.getSource().getTypeClassName().isEmpty())) {
String typeArg =
- getSourceType(functionDetailsBuilder.getSource().getClassName(), classLoader).getName();
+ getSourceType(functionDetailsBuilder.getSource().getClassName(), typePool).asErasure()
+ .getTypeName();
Function.SourceSpec.Builder sourceBuilder =
Function.SourceSpec.newBuilder(functionDetailsBuilder.getSource());
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java
index 26651010fef..47bd3ba0472 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java
@@ -121,17 +121,17 @@ public class ThreadRuntime implements Runtime {
if (componentType == Function.FunctionDetails.ComponentType.FUNCTION && functionsManager.isPresent()) {
return functionsManager.get()
.getFunction(instanceConfig.getFunctionDetails().getBuiltin())
- .getClassLoader();
+ .getFunctionPackage().getClassLoader();
}
if (componentType == Function.FunctionDetails.ComponentType.SOURCE && connectorsManager.isPresent()) {
return connectorsManager.get()
.getConnector(instanceConfig.getFunctionDetails().getSource().getBuiltin())
- .getClassLoader();
+ .getConnectorFunctionPackage().getClassLoader();
}
if (componentType == Function.FunctionDetails.ComponentType.SINK && connectorsManager.isPresent()) {
return connectorsManager.get()
.getConnector(instanceConfig.getFunctionDetails().getSink().getBuiltin())
- .getClassLoader();
+ .getConnectorFunctionPackage().getClassLoader();
}
}
return loadJars(jarFile, instanceConfig, instanceConfig.getFunctionDetails().getName(),
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java
index 643afa4f26c..c78bb734285 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java
@@ -18,6 +18,7 @@
*/
package org.apache.pulsar.functions.worker;
+import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
@@ -27,18 +28,35 @@ import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.common.io.ConfigFieldDefinition;
import org.apache.pulsar.common.io.ConnectorDefinition;
+import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory;
import org.apache.pulsar.functions.utils.io.Connector;
import org.apache.pulsar.functions.utils.io.ConnectorUtils;
@Slf4j
-public class ConnectorsManager {
+public class ConnectorsManager implements AutoCloseable {
@Getter
private volatile TreeMap<String, Connector> connectors;
+ @VisibleForTesting
+ public ConnectorsManager() {
+ this.connectors = new TreeMap<>();
+ }
+
public ConnectorsManager(WorkerConfig workerConfig) throws IOException {
- this.connectors = ConnectorUtils
- .searchForConnectors(workerConfig.getConnectorsDirectory(), workerConfig.getNarExtractionDirectory());
+ this.connectors = createConnectors(workerConfig);
+ }
+
+ private static TreeMap<String, Connector> createConnectors(WorkerConfig workerConfig) throws IOException {
+ boolean enableClassloading = workerConfig.getEnableClassloadingOfBuiltinFiles()
+ || ThreadRuntimeFactory.class.getName().equals(workerConfig.getFunctionRuntimeFactoryClassName());
+ return ConnectorUtils.searchForConnectors(workerConfig.getConnectorsDirectory(),
+ workerConfig.getNarExtractionDirectory(), enableClassloading);
+ }
+
+ @VisibleForTesting
+ public void addConnector(String connectorType, Connector connector) {
+ connectors.put(connectorType, connector);
}
public Connector getConnector(String connectorType) {
@@ -71,7 +89,25 @@ public class ConnectorsManager {
}
public void reloadConnectors(WorkerConfig workerConfig) throws IOException {
- connectors = ConnectorUtils
- .searchForConnectors(workerConfig.getConnectorsDirectory(), workerConfig.getNarExtractionDirectory());
+ TreeMap<String, Connector> oldConnectors = connectors;
+ this.connectors = createConnectors(workerConfig);
+ closeConnectors(oldConnectors);
}
+
+ @Override
+ public void close() {
+ closeConnectors(connectors);
+ }
+
+ private void closeConnectors(TreeMap<String, Connector> connectorMap) {
+ connectorMap.values().forEach(connector -> {
+ try {
+ connector.close();
+ } catch (Exception e) {
+ log.warn("Failed to close connector", e);
+ }
+ });
+ connectorMap.clear();
+ }
+
}
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java
index 0f1c0fcc835..753d41f0fa7 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java
@@ -18,20 +18,30 @@
*/
package org.apache.pulsar.functions.worker;
+import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
import java.nio.file.Path;
import java.util.TreeMap;
import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory;
import org.apache.pulsar.functions.utils.functions.FunctionArchive;
import org.apache.pulsar.functions.utils.functions.FunctionUtils;
@Slf4j
-public class FunctionsManager {
-
+public class FunctionsManager implements AutoCloseable {
private TreeMap<String, FunctionArchive> functions;
+ @VisibleForTesting
+ public FunctionsManager() {
+ this.functions = new TreeMap<>();
+ }
+
public FunctionsManager(WorkerConfig workerConfig) throws IOException {
- this.functions = FunctionUtils.searchForFunctions(workerConfig.getFunctionsDirectory());
+ this.functions = createFunctions(workerConfig);
+ }
+
+ public void addFunction(String functionType, FunctionArchive functionArchive) {
+ functions.put(functionType, functionArchive);
}
public FunctionArchive getFunction(String functionType) {
@@ -43,6 +53,32 @@ public class FunctionsManager {
}
public void reloadFunctions(WorkerConfig workerConfig) throws IOException {
- this.functions = FunctionUtils.searchForFunctions(workerConfig.getFunctionsDirectory());
+ TreeMap<String, FunctionArchive> oldFunctions = functions;
+ this.functions = createFunctions(workerConfig);
+ closeFunctions(oldFunctions);
+ }
+
+ private static TreeMap<String, FunctionArchive> createFunctions(WorkerConfig workerConfig) throws IOException {
+ boolean enableClassloading = workerConfig.getEnableClassloadingOfBuiltinFiles()
+ || ThreadRuntimeFactory.class.getName().equals(workerConfig.getFunctionRuntimeFactoryClassName());
+ return FunctionUtils.searchForFunctions(workerConfig.getFunctionsDirectory(),
+ workerConfig.getNarExtractionDirectory(),
+ enableClassloading);
+ }
+
+ @Override
+ public void close() {
+ closeFunctions(functions);
+ }
+
+ private void closeFunctions(TreeMap<String, FunctionArchive> functionMap) {
+ functionMap.values().forEach(functionArchive -> {
+ try {
+ functionArchive.close();
+ } catch (Exception e) {
+ log.warn("Failed to close function archive", e);
+ }
+ });
+ functionMap.clear();
}
}
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
index 76078a46e79..5432d5c4610 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
@@ -225,6 +225,22 @@ public class WorkerConfig implements Serializable, PulsarConfiguration {
)
private int zooKeeperCacheExpirySeconds = -1;
+ @FieldContext(
+ category = CATEGORY_WORKER,
+ doc = "Specifies if the function worker should use classloading for validating submissions for built-in "
+ + "connectors and functions. This is required for validateConnectorConfig to take effect. "
+ + "Default is false."
+ )
+ private Boolean enableClassloadingOfBuiltinFiles = false;
+
+ @FieldContext(
+ category = CATEGORY_WORKER,
+ doc = "Specifies if the function worker should use classloading for validating submissions for external "
+ + "connectors and functions. This is required for validateConnectorConfig to take effect. "
+ + "Default is false."
+ )
+ private Boolean enableClassloadingOfExternalFiles = false;
+
@FieldContext(
category = CATEGORY_CONNECTORS,
doc = "The path to the location to locate builtin connectors"
@@ -237,7 +253,10 @@ public class WorkerConfig implements Serializable, PulsarConfiguration {
private String narExtractionDirectory = NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR;
@FieldContext(
category = CATEGORY_CONNECTORS,
- doc = "Should we validate connector config during submission"
+ doc = "Enables extended validation for connector config with fine-grain annotation based validation "
+ + "during submission. Classloading with either enableClassloadingOfExternalFiles or "
+ + "enableClassloadingOfBuiltinFiles must be enabled on the worker for this to take effect. "
+ + "Default is false."
)
private Boolean validateConnectorConfig = false;
@FieldContext(
diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml
index 27274258a18..befc887f67c 100644
--- a/pulsar-functions/utils/pom.xml
+++ b/pulsar-functions/utils/pom.xml
@@ -87,6 +87,17 @@
<artifactId>typetools</artifactId>
</dependency>
+ <dependency>
+ <groupId>net.bytebuddy</groupId>
+ <artifactId>byte-buddy</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.zeroturnaround</groupId>
+ <artifactId>zt-zip</artifactId>
+ <version>1.17</version>
+ </dependency>
+
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>pulsar-client-original</artifactId>
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
index 49b71afcad7..0e17907fb3b 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
@@ -22,27 +22,25 @@ import static org.apache.commons.lang3.StringUtils.isEmpty;
import com.google.protobuf.AbstractMessage.Builder;
import com.google.protobuf.MessageOrBuilder;
import com.google.protobuf.util.JsonFormat;
-import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.io.ObjectOutputStream;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.ParameterizedType;
-import java.lang.reflect.Type;
-import java.net.MalformedURLException;
import java.net.ServerSocket;
import java.net.URISyntaxException;
import java.net.URL;
+import java.net.URLConnection;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.Collection;
import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
-import net.jodah.typetools.TypeResolver;
+import net.bytebuddy.description.type.TypeDefinition;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.description.type.TypeList;
+import net.bytebuddy.pool.TypePool;
import org.apache.commons.lang3.StringUtils;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.SubscriptionInitialPosition;
@@ -50,15 +48,11 @@ import org.apache.pulsar.client.impl.MessageIdImpl;
import org.apache.pulsar.client.impl.TopicMessageIdImpl;
import org.apache.pulsar.common.functions.FunctionConfig;
import org.apache.pulsar.common.functions.Utils;
-import org.apache.pulsar.common.nar.NarClassLoader;
-import org.apache.pulsar.common.nar.NarClassLoaderBuilder;
-import org.apache.pulsar.common.util.ClassLoaderUtils;
import org.apache.pulsar.functions.api.Function;
import org.apache.pulsar.functions.api.Record;
import org.apache.pulsar.functions.api.WindowFunction;
+import org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType;
import org.apache.pulsar.functions.proto.Function.FunctionDetails.Runtime;
-import org.apache.pulsar.functions.utils.functions.FunctionUtils;
-import org.apache.pulsar.functions.utils.io.ConnectorUtils;
import org.apache.pulsar.io.core.BatchSource;
import org.apache.pulsar.io.core.Sink;
import org.apache.pulsar.io.core.Source;
@@ -92,82 +86,79 @@ public class FunctionCommon {
}
}
- public static Class<?>[] getFunctionTypes(FunctionConfig functionConfig, ClassLoader classLoader)
+ public static TypeDefinition[] getFunctionTypes(FunctionConfig functionConfig, TypePool typePool)
throws ClassNotFoundException {
- return getFunctionTypes(functionConfig, classLoader.loadClass(functionConfig.getClassName()));
+ return getFunctionTypes(functionConfig, typePool.describe(functionConfig.getClassName()).resolve());
}
- public static Class<?>[] getFunctionTypes(FunctionConfig functionConfig, Class functionClass)
- throws ClassNotFoundException {
+ public static TypeDefinition[] getFunctionTypes(FunctionConfig functionConfig, TypeDefinition functionClass) {
boolean isWindowConfigPresent = functionConfig.getWindowConfig() != null;
return getFunctionTypes(functionClass, isWindowConfigPresent);
}
- public static Class<?>[] getFunctionTypes(Class userClass, boolean isWindowConfigPresent) {
- Class<?>[] typeArgs;
+ public static TypeDefinition[] getFunctionTypes(TypeDefinition userClass, boolean isWindowConfigPresent) {
+ Class<?> classParent = getFunctionClassParent(userClass, isWindowConfigPresent);
+ TypeList.Generic typeArgsList = resolveInterfaceTypeArguments(userClass, classParent);
+ TypeDescription.Generic[] typeArgs = new TypeDescription.Generic[2];
+ typeArgs[0] = typeArgsList.get(0);
+ typeArgs[1] = typeArgsList.get(1);
// if window function
if (isWindowConfigPresent) {
- if (WindowFunction.class.isAssignableFrom(userClass)) {
- typeArgs = getFunctionTypesUnwrappingRecordIfNeeded(WindowFunction.class, userClass);
- } else {
- typeArgs = getFunctionTypesUnwrappingRecordIfNeeded(java.util.function.Function.class, userClass);
- if (!typeArgs[0].equals(Collection.class)) {
+ if (classParent.equals(java.util.function.Function.class)) {
+ if (!typeArgs[0].asErasure().isAssignableTo(Collection.class)) {
throw new IllegalArgumentException("Window function must take a collection as input");
}
- Type type = TypeResolver.resolveGenericType(java.util.function.Function.class, userClass);
- Type collectionType = ((ParameterizedType) type).getActualTypeArguments()[0];
- Type actualInputType = ((ParameterizedType) collectionType).getActualTypeArguments()[0];
- typeArgs[0] = (Class<?>) actualInputType;
- }
- } else {
- if (Function.class.isAssignableFrom(userClass)) {
- typeArgs = getFunctionTypesUnwrappingRecordIfNeeded(Function.class, userClass);
- } else {
- typeArgs = getFunctionTypesUnwrappingRecordIfNeeded(java.util.function.Function.class, userClass);
+ typeArgs[0] = typeArgs[0].getTypeArguments().get(0);
}
}
-
+ if (typeArgs[1].asErasure().isAssignableTo(Record.class)) {
+ typeArgs[1] = typeArgs[1].getTypeArguments().get(0);
+ }
+ if (typeArgs[1].asErasure().isAssignableTo(CompletableFuture.class)) {
+ typeArgs[1] = typeArgs[1].getTypeArguments().get(0);
+ }
return typeArgs;
}
- private static Class<?>[] getFunctionTypesUnwrappingRecordIfNeeded(Class<?> type, Class<?> subType) {
- Class<?>[] typeArgs = TypeResolver.resolveRawArguments(type, subType);
- if (typeArgs[1].equals(Record.class)) {
- Type genericType = TypeResolver.resolveGenericType(type, subType);
- Type recordType = ((ParameterizedType) genericType).getActualTypeArguments()[1];
- Type actualInputType = ((ParameterizedType) recordType).getActualTypeArguments()[0];
- typeArgs[1] = (Class<?>) actualInputType;
+ private static TypeList.Generic resolveInterfaceTypeArguments(TypeDefinition userClass, Class<?> interfaceClass) {
+ if (!interfaceClass.isInterface()) {
+ throw new IllegalArgumentException("interfaceClass must be an interface");
+ }
+ for (TypeDescription.Generic interfaze : userClass.getInterfaces()) {
+ if (interfaze.asErasure().isAssignableTo(interfaceClass)) {
+ return interfaze.getTypeArguments();
+ }
+ }
+ if (userClass.getSuperClass() != null) {
+ return resolveInterfaceTypeArguments(userClass.getSuperClass(), interfaceClass);
}
+ return null;
+ }
+
+ public static TypeDescription.Generic[] getRawFunctionTypes(TypeDefinition userClass,
+ boolean isWindowConfigPresent) {
+ Class<?> classParent = getFunctionClassParent(userClass, isWindowConfigPresent);
+ TypeList.Generic typeArgsList = resolveInterfaceTypeArguments(userClass, classParent);
+ TypeDescription.Generic[] typeArgs = new TypeDescription.Generic[2];
+ typeArgs[0] = typeArgsList.get(0);
+ typeArgs[1] = typeArgsList.get(1);
return typeArgs;
}
- public static Object createInstance(String userClassName, ClassLoader classLoader) {
- Class<?> theCls;
- try {
- theCls = Class.forName(userClassName);
- } catch (ClassNotFoundException | NoClassDefFoundError cnfe) {
- try {
- theCls = Class.forName(userClassName, true, classLoader);
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
- throw new RuntimeException("User class must be in class path", cnfe);
+ public static Class<?> getFunctionClassParent(TypeDefinition userClass, boolean isWindowConfigPresent) {
+ if (isWindowConfigPresent) {
+ if (userClass.asErasure().isAssignableTo(WindowFunction.class)) {
+ return WindowFunction.class;
+ } else {
+ return java.util.function.Function.class;
+ }
+ } else {
+ if (userClass.asErasure().isAssignableTo(Function.class)) {
+ return Function.class;
+ } else {
+ return java.util.function.Function.class;
}
}
- Object result;
- try {
- Constructor<?> meth = theCls.getDeclaredConstructor();
- meth.setAccessible(true);
- result = meth.newInstance();
- } catch (InstantiationException ie) {
- throw new RuntimeException("User class must be concrete", ie);
- } catch (NoSuchMethodException e) {
- throw new RuntimeException("User class doesn't have such method", e);
- } catch (IllegalAccessException e) {
- throw new RuntimeException("User class must have a no-arg constructor", e);
- } catch (InvocationTargetException e) {
- throw new RuntimeException("User class constructor throws exception", e);
- }
- return result;
-
}
public static Runtime convertRuntime(FunctionConfig.Runtime runtime) {
@@ -210,50 +201,46 @@ public class FunctionCommon {
throw new RuntimeException("Unrecognized processing guarantee: " + processingGuarantees.name());
}
- public static Class<?> getSourceType(String className, ClassLoader classLoader) throws ClassNotFoundException {
- return getSourceType(classLoader.loadClass(className));
+ public static TypeDefinition getSourceType(String className, TypePool typePool) {
+ return getSourceType(typePool.describe(className).resolve());
}
- public static Class<?> getSourceType(Class sourceClass) {
-
- if (Source.class.isAssignableFrom(sourceClass)) {
- return TypeResolver.resolveRawArgument(Source.class, sourceClass);
- } else if (BatchSource.class.isAssignableFrom(sourceClass)) {
- return TypeResolver.resolveRawArgument(BatchSource.class, sourceClass);
+ public static TypeDefinition getSourceType(TypeDefinition sourceClass) {
+ if (sourceClass.asErasure().isAssignableTo(Source.class)) {
+ return resolveInterfaceTypeArguments(sourceClass, Source.class).get(0);
+ } else if (sourceClass.asErasure().isAssignableTo(BatchSource.class)) {
+ return resolveInterfaceTypeArguments(sourceClass, BatchSource.class).get(0);
} else {
throw new IllegalArgumentException(
String.format("Source class %s does not implement the correct interface",
- sourceClass.getName()));
+ sourceClass.getActualName()));
}
}
- public static Class<?> getSinkType(String className, ClassLoader classLoader) throws ClassNotFoundException {
- return getSinkType(classLoader.loadClass(className));
+ public static TypeDefinition getSinkType(String className, TypePool typePool) {
+ return getSinkType(typePool.describe(className).resolve());
}
- public static Class<?> getSinkType(Class sinkClass) {
- return TypeResolver.resolveRawArgument(Sink.class, sinkClass);
+ public static TypeDefinition getSinkType(TypeDefinition sinkClass) {
+ if (sinkClass.asErasure().isAssignableTo(Sink.class)) {
+ return resolveInterfaceTypeArguments(sinkClass, Sink.class).get(0);
+ } else {
+ throw new IllegalArgumentException(
+ String.format("Sink class %s does not implement the correct interface",
+ sinkClass.getActualName()));
+ }
}
public static void downloadFromHttpUrl(String destPkgUrl, File targetFile) throws IOException {
- URL website = new URL(destPkgUrl);
- try (InputStream in = website.openStream()) {
+ final URL url = new URL(destPkgUrl);
+ final URLConnection connection = url.openConnection();
+ try (InputStream in = connection.getInputStream()) {
log.info("Downloading function package from {} to {} ...", destPkgUrl, targetFile.getAbsoluteFile());
Files.copy(in, targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
}
log.info("Downloading function package from {} to {} completed!", destPkgUrl, targetFile.getAbsoluteFile());
}
- public static ClassLoader extractClassLoader(String destPkgUrl) throws IOException, URISyntaxException {
- File file = extractFileFromPkgURL(destPkgUrl);
- try {
- return ClassLoaderUtils.loadJar(file);
- } catch (MalformedURLException e) {
- throw new IllegalArgumentException(
- "Corrupt User PackageFile " + file + " with error " + e.getMessage());
- }
- }
-
public static File createPkgTempFile() throws IOException {
return File.createTempFile("functions", ".tmp");
}
@@ -277,21 +264,6 @@ public class FunctionCommon {
}
}
- public static NarClassLoader extractNarClassLoader(File packageFile,
- String narExtractionDirectory) {
- if (packageFile != null) {
- try {
- return NarClassLoaderBuilder.builder()
- .narFile(packageFile)
- .extractionDirectory(narExtractionDirectory)
- .build();
- } catch (IOException e) {
- throw new IllegalArgumentException(e.getMessage());
- }
- }
- return null;
- }
-
public static String getFullyQualifiedInstanceId(org.apache.pulsar.functions.proto.Function.Instance instance) {
return getFullyQualifiedInstanceId(
instance.getFunctionMetaData().getFunctionDetails().getTenant(),
@@ -327,17 +299,6 @@ public class FunctionCommon {
return new MessageIdImpl(ledgerId, entryId, -1);
}
- public static byte[] toByteArray(Object obj) throws IOException {
- byte[] bytes = null;
- try (ByteArrayOutputStream bos = new ByteArrayOutputStream();
- ObjectOutputStream oos = new ObjectOutputStream(bos)) {
- oos.writeObject(obj);
- oos.flush();
- bytes = bos.toByteArray();
- }
- return bytes;
- }
-
public static String getUniquePackageName(String packageName) {
return String.format("%s-%s", UUID.randomUUID().toString(), packageName);
}
@@ -385,173 +346,38 @@ public class FunctionCommon {
throw new RuntimeException("Invalid Fully Qualified Function Name " + fqfn);
}
- public static Class<?> getTypeArg(String className, Class<?> funClass, ClassLoader classLoader)
- throws ClassNotFoundException {
- Class<?> loadedClass = classLoader.loadClass(className);
- if (!funClass.isAssignableFrom(loadedClass)) {
- throw new IllegalArgumentException(
- String.format("class %s is not type of %s", className, funClass.getName()));
- }
- return TypeResolver.resolveRawArgument(funClass, loadedClass);
- }
-
public static double roundDecimal(double value, int places) {
double scale = Math.pow(10, places);
return Math.round(value * scale) / scale;
}
- public static ClassLoader getClassLoaderFromPackage(
- org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType componentType,
- String className,
- File packageFile,
- String narExtractionDirectory) {
- String connectorClassName = className;
- ClassLoader jarClassLoader = null;
- boolean keepJarClassLoader = false;
- ClassLoader narClassLoader = null;
- boolean keepNarClassLoader = false;
-
- Exception jarClassLoaderException = null;
- Exception narClassLoaderException = null;
-
- try {
- try {
- jarClassLoader = ClassLoaderUtils.extractClassLoader(packageFile);
- } catch (Exception e) {
- jarClassLoaderException = e;
- }
- try {
- narClassLoader = FunctionCommon.extractNarClassLoader(packageFile, narExtractionDirectory);
- } catch (Exception e) {
- narClassLoaderException = e;
- }
-
- // if connector class name is not provided, we can only try to load archive as a NAR
- if (isEmpty(connectorClassName)) {
- if (narClassLoader == null) {
- throw new IllegalArgumentException(String.format("%s package does not have the correct format. "
- + "Pulsar cannot determine if the package is a NAR package or JAR package. "
- + "%s classname is not provided and attempts to load it as a NAR package produced "
- + "the following error.",
- capFirstLetter(componentType), capFirstLetter(componentType)),
- narClassLoaderException);
- }
- try {
- if (componentType
- == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.FUNCTION) {
- connectorClassName = FunctionUtils.getFunctionClass((NarClassLoader) narClassLoader);
- } else if (componentType
- == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SOURCE) {
- connectorClassName = ConnectorUtils.getIOSourceClass((NarClassLoader) narClassLoader);
- } else {
- connectorClassName = ConnectorUtils.getIOSinkClass((NarClassLoader) narClassLoader);
- }
- } catch (IOException e) {
- throw new IllegalArgumentException(String.format("Failed to extract %s class from archive",
- componentType.toString().toLowerCase()), e);
- }
-
- try {
- narClassLoader.loadClass(connectorClassName);
- keepNarClassLoader = true;
- return narClassLoader;
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
- throw new IllegalArgumentException(
- String.format("%s class %s must be in class path", capFirstLetter(componentType),
- connectorClassName), e);
- }
-
- } else {
- // if connector class name is provided, we need to try to load it as a JAR and as a NAR.
- if (jarClassLoader != null) {
- try {
- jarClassLoader.loadClass(connectorClassName);
- keepJarClassLoader = true;
- return jarClassLoader;
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
- // class not found in JAR try loading as a NAR and searching for the class
- if (narClassLoader != null) {
-
- try {
- narClassLoader.loadClass(connectorClassName);
- keepNarClassLoader = true;
- return narClassLoader;
- } catch (ClassNotFoundException | NoClassDefFoundError e1) {
- throw new IllegalArgumentException(
- String.format("%s class %s must be in class path",
- capFirstLetter(componentType), connectorClassName), e1);
- }
- } else {
- throw new IllegalArgumentException(
- String.format("%s class %s must be in class path", capFirstLetter(componentType),
- connectorClassName), e);
- }
- }
- } else if (narClassLoader != null) {
- try {
- narClassLoader.loadClass(connectorClassName);
- keepNarClassLoader = true;
- return narClassLoader;
- } catch (ClassNotFoundException | NoClassDefFoundError e1) {
- throw new IllegalArgumentException(
- String.format("%s class %s must be in class path",
- capFirstLetter(componentType), connectorClassName), e1);
- }
- } else {
- StringBuilder errorMsg = new StringBuilder(capFirstLetter(componentType)
- + " package does not have the correct format."
- + " Pulsar cannot determine if the package is a NAR package or JAR package.");
-
- if (jarClassLoaderException != null) {
- errorMsg.append(
- " Attempts to load it as a JAR package produced error: " + jarClassLoaderException
- .getMessage());
- }
-
- if (narClassLoaderException != null) {
- errorMsg.append(
- " Attempts to load it as a NAR package produced error: " + narClassLoaderException
- .getMessage());
- }
-
- throw new IllegalArgumentException(errorMsg.toString());
- }
- }
- } finally {
- if (!keepJarClassLoader) {
- ClassLoaderUtils.closeClassLoader(jarClassLoader);
- }
- if (!keepNarClassLoader) {
- ClassLoaderUtils.closeClassLoader(narClassLoader);
- }
- }
- }
-
public static String capFirstLetter(Enum en) {
return StringUtils.capitalize(en.toString().toLowerCase());
}
public static boolean isFunctionCodeBuiltin(
- org.apache.pulsar.functions.proto.Function.FunctionDetailsOrBuilder functionDetails) {
- if (functionDetails.hasSource()) {
+ org.apache.pulsar.functions.proto.Function.FunctionDetailsOrBuilder functionDetail) {
+ return isFunctionCodeBuiltin(functionDetail, functionDetail.getComponentType());
+ }
+
+ public static boolean isFunctionCodeBuiltin(
+ org.apache.pulsar.functions.proto.Function.FunctionDetailsOrBuilder functionDetails,
+ ComponentType componentType) {
+ if (componentType == ComponentType.SOURCE && functionDetails.hasSource()) {
org.apache.pulsar.functions.proto.Function.SourceSpec sourceSpec = functionDetails.getSource();
if (!isEmpty(sourceSpec.getBuiltin())) {
return true;
}
}
- if (functionDetails.hasSink()) {
+ if (componentType == ComponentType.SINK && functionDetails.hasSink()) {
org.apache.pulsar.functions.proto.Function.SinkSpec sinkSpec = functionDetails.getSink();
if (!isEmpty(sinkSpec.getBuiltin())) {
return true;
}
}
- if (!isEmpty(functionDetails.getBuiltin())) {
- return true;
- }
-
- return false;
+ return componentType == ComponentType.FUNCTION && !isEmpty(functionDetails.getBuiltin());
}
public static SubscriptionInitialPosition convertFromFunctionDetailsSubscriptionPosition(
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
index a8630dd3b34..dcd6982315e 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
@@ -18,21 +18,18 @@
*/
package org.apache.pulsar.functions.utils;
-import static org.apache.commons.lang.StringUtils.isBlank;
-import static org.apache.commons.lang.StringUtils.isNotBlank;
-import static org.apache.commons.lang.StringUtils.isNotEmpty;
+import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isEmpty;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+import static org.apache.commons.lang3.StringUtils.isNotEmpty;
import static org.apache.pulsar.common.functions.Utils.BUILTIN;
-import static org.apache.pulsar.common.util.ClassLoaderUtils.loadJar;
import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsSubscriptionPosition;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import java.io.File;
-import java.io.IOException;
import java.lang.reflect.Type;
-import java.net.MalformedURLException;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
@@ -42,10 +39,13 @@ import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang.StringUtils;
+import net.bytebuddy.description.type.TypeDefinition;
+import net.bytebuddy.pool.TypePool;
+import org.apache.commons.lang3.StringUtils;
import org.apache.pulsar.client.api.SubscriptionInitialPosition;
import org.apache.pulsar.common.functions.ConsumerConfig;
import org.apache.pulsar.common.functions.FunctionConfig;
+import org.apache.pulsar.common.functions.FunctionDefinition;
import org.apache.pulsar.common.functions.ProducerConfig;
import org.apache.pulsar.common.functions.Resources;
import org.apache.pulsar.common.functions.WindowConfig;
@@ -53,7 +53,6 @@ import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.util.ObjectMapperFactory;
import org.apache.pulsar.functions.proto.Function;
import org.apache.pulsar.functions.proto.Function.FunctionDetails;
-import org.apache.pulsar.functions.utils.functions.FunctionUtils;
@Slf4j
public class FunctionConfigUtils {
@@ -72,26 +71,21 @@ public class FunctionConfigUtils {
private static final ObjectMapper OBJECT_MAPPER = ObjectMapperFactory.create();
- public static FunctionDetails convert(FunctionConfig functionConfig, ClassLoader classLoader)
- throws IllegalArgumentException {
+ public static FunctionDetails convert(FunctionConfig functionConfig) {
+ return convert(functionConfig, (ValidatableFunctionPackage) null);
+ }
- if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
- if (classLoader != null) {
- try {
- Class<?>[] typeArgs = FunctionCommon.getFunctionTypes(functionConfig, classLoader);
- return convert(
- functionConfig,
- new ExtractedFunctionDetails(
- functionConfig.getClassName(),
- typeArgs[0].getName(),
- typeArgs[1].getName()));
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
- throw new IllegalArgumentException(
- String.format("Function class %s must be in class path", functionConfig.getClassName()), e);
- }
- }
+ public static FunctionDetails convert(FunctionConfig functionConfig,
+ ValidatableFunctionPackage validatableFunctionPackage)
+ throws IllegalArgumentException {
+ if (functionConfig == null) {
+ throw new IllegalArgumentException("Function config is not provided");
+ }
+ if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA && validatableFunctionPackage != null) {
+ return convert(functionConfig, doJavaChecks(functionConfig, validatableFunctionPackage));
+ } else {
+ return convert(functionConfig, new ExtractedFunctionDetails(functionConfig.getClassName(), null, null));
}
- return convert(functionConfig, new ExtractedFunctionDetails(functionConfig.getClassName(), null, null));
}
public static FunctionDetails convert(FunctionConfig functionConfig, ExtractedFunctionDetails extractedDetails)
@@ -568,7 +562,6 @@ public class FunctionConfigUtils {
} else if (functionConfig.getGo() != null) {
functionConfig.setRuntime(FunctionConfig.Runtime.GO);
}
-
WindowConfig windowConfig = functionConfig.getWindowConfig();
if (windowConfig != null) {
WindowConfigUtils.inferMissingArguments(windowConfig);
@@ -576,48 +569,49 @@ public class FunctionConfigUtils {
}
}
- private static ExtractedFunctionDetails doJavaChecks(FunctionConfig functionConfig, ClassLoader clsLoader) {
+ public static ExtractedFunctionDetails doJavaChecks(FunctionConfig functionConfig,
+ ValidatableFunctionPackage validatableFunctionPackage) {
- String functionClassName = functionConfig.getClassName();
- Class functionClass;
+ String functionClassName = StringUtils.trimToNull(functionConfig.getClassName());
+ TypeDefinition functionClass;
try {
// if class name in function config is not set, this should be a built-in function
// thus we should try to find its class name in the NAR service definition
if (functionClassName == null) {
- try {
- functionClassName = FunctionUtils.getFunctionClass(clsLoader);
- } catch (IOException e) {
- throw new IllegalArgumentException("Failed to extract source class from archive", e);
+ FunctionDefinition functionDefinition =
+ validatableFunctionPackage.getFunctionMetaData(FunctionDefinition.class);
+ if (functionDefinition == null) {
+ throw new IllegalArgumentException("Function class name is not provided.");
+ }
+ functionClassName = functionDefinition.getFunctionClass();
+ if (functionClassName == null) {
+ throw new IllegalArgumentException("Function class name is not provided.");
}
}
- functionClass = clsLoader.loadClass(functionClassName);
+ functionClass = validatableFunctionPackage.resolveType(functionClassName);
- if (!org.apache.pulsar.functions.api.Function.class.isAssignableFrom(functionClass)
- && !java.util.function.Function.class.isAssignableFrom(functionClass)
- && !org.apache.pulsar.functions.api.WindowFunction.class.isAssignableFrom(functionClass)) {
+ if (!functionClass.asErasure().isAssignableTo(org.apache.pulsar.functions.api.Function.class)
+ && !functionClass.asErasure().isAssignableTo(java.util.function.Function.class)
+ && !functionClass.asErasure()
+ .isAssignableTo(org.apache.pulsar.functions.api.WindowFunction.class)) {
throw new IllegalArgumentException(
String.format("Function class %s does not implement the correct interface",
- functionClass.getName()));
+ functionClassName));
}
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
+ } catch (TypePool.Resolution.NoSuchTypeException e) {
throw new IllegalArgumentException(
- String.format("Function class %s must be in class path", functionConfig.getClassName()), e);
+ String.format("Function class %s must be in class path", functionClassName), e);
}
- Class<?>[] typeArgs;
- try {
- typeArgs = FunctionCommon.getFunctionTypes(functionConfig, functionClass);
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
- throw new IllegalArgumentException(
- String.format("Function class %s must be in class path", functionConfig.getClassName()), e);
- }
+ TypeDefinition[] typeArgs = FunctionCommon.getFunctionTypes(functionConfig, functionClass);
// inputs use default schema, so there is no check needed there
// Check if the Input serialization/deserialization class exists in jar or already loaded and that it
// implements SerDe class
if (functionConfig.getCustomSerdeInputs() != null) {
functionConfig.getCustomSerdeInputs().forEach((topicName, inputSerializer) -> {
- ValidatorUtils.validateSerde(inputSerializer, typeArgs[0], clsLoader, true);
+ ValidatorUtils.validateSerde(inputSerializer, typeArgs[0], validatableFunctionPackage.getTypePool(),
+ true);
});
}
@@ -632,8 +626,8 @@ public class FunctionConfigUtils {
throw new IllegalArgumentException(
String.format("Topic %s has an incorrect schema Info", topicName));
}
- ValidatorUtils.validateSchema(consumerConfig.getSchemaType(), typeArgs[0], clsLoader, true);
-
+ ValidatorUtils.validateSchema(consumerConfig.getSchemaType(), typeArgs[0],
+ validatableFunctionPackage.getTypePool(), true);
});
}
@@ -648,13 +642,16 @@ public class FunctionConfigUtils {
"Only one of schemaType or serdeClassName should be set in inputSpec");
}
if (!isEmpty(conf.getSerdeClassName())) {
- ValidatorUtils.validateSerde(conf.getSerdeClassName(), typeArgs[0], clsLoader, true);
+ ValidatorUtils.validateSerde(conf.getSerdeClassName(), typeArgs[0],
+ validatableFunctionPackage.getTypePool(), true);
}
if (!isEmpty(conf.getSchemaType())) {
- ValidatorUtils.validateSchema(conf.getSchemaType(), typeArgs[0], clsLoader, true);
+ ValidatorUtils.validateSchema(conf.getSchemaType(), typeArgs[0],
+ validatableFunctionPackage.getTypePool(), true);
}
if (conf.getCryptoConfig() != null) {
- ValidatorUtils.validateCryptoKeyReader(conf.getCryptoConfig(), clsLoader, false);
+ ValidatorUtils.validateCryptoKeyReader(conf.getCryptoConfig(),
+ validatableFunctionPackage.getTypePool(), false);
}
});
}
@@ -662,8 +659,8 @@ public class FunctionConfigUtils {
if (Void.class.equals(typeArgs[1])) {
return new FunctionConfigUtils.ExtractedFunctionDetails(
functionClassName,
- typeArgs[0].getName(),
- typeArgs[1].getName());
+ typeArgs[0].asErasure().getTypeName(),
+ typeArgs[1].asErasure().getTypeName());
}
// One and only one of outputSchemaType and outputSerdeClassName should be set
@@ -673,22 +670,25 @@ public class FunctionConfigUtils {
}
if (!isEmpty(functionConfig.getOutputSchemaType())) {
- ValidatorUtils.validateSchema(functionConfig.getOutputSchemaType(), typeArgs[1], clsLoader, false);
+ ValidatorUtils.validateSchema(functionConfig.getOutputSchemaType(), typeArgs[1],
+ validatableFunctionPackage.getTypePool(), false);
}
if (!isEmpty(functionConfig.getOutputSerdeClassName())) {
- ValidatorUtils.validateSerde(functionConfig.getOutputSerdeClassName(), typeArgs[1], clsLoader, false);
+ ValidatorUtils.validateSerde(functionConfig.getOutputSerdeClassName(), typeArgs[1],
+ validatableFunctionPackage.getTypePool(), false);
}
if (functionConfig.getProducerConfig() != null
&& functionConfig.getProducerConfig().getCryptoConfig() != null) {
ValidatorUtils
- .validateCryptoKeyReader(functionConfig.getProducerConfig().getCryptoConfig(), clsLoader, true);
+ .validateCryptoKeyReader(functionConfig.getProducerConfig().getCryptoConfig(),
+ validatableFunctionPackage.getTypePool(), true);
}
return new FunctionConfigUtils.ExtractedFunctionDetails(
functionClassName,
- typeArgs[0].getName(),
- typeArgs[1].getName());
+ typeArgs[0].asErasure().getTypeName(),
+ typeArgs[1].asErasure().getTypeName());
}
private static void doPythonChecks(FunctionConfig functionConfig) {
@@ -703,10 +703,6 @@ public class FunctionConfigUtils {
if (functionConfig.getMaxMessageRetries() != null && functionConfig.getMaxMessageRetries() >= 0) {
throw new IllegalArgumentException("Message retries not yet supported in python");
}
-
- if (functionConfig.getRetainKeyOrdering() != null && functionConfig.getRetainKeyOrdering()) {
- throw new IllegalArgumentException("Retain Key Orderering not yet supported in python");
- }
}
private static void doGolangChecks(FunctionConfig functionConfig) {
@@ -737,7 +733,7 @@ public class FunctionConfigUtils {
}
}
- private static void doCommonChecks(FunctionConfig functionConfig) {
+ public static void doCommonChecks(FunctionConfig functionConfig) {
if (isEmpty(functionConfig.getTenant())) {
throw new IllegalArgumentException("Function tenant cannot be null");
}
@@ -879,7 +875,7 @@ public class FunctionConfigUtils {
}
}
- private static Collection<String> collectAllInputTopics(FunctionConfig functionConfig) {
+ public static Collection<String> collectAllInputTopics(FunctionConfig functionConfig) {
List<String> retval = new LinkedList<>();
if (functionConfig.getInputs() != null) {
retval.addAll(functionConfig.getInputs());
@@ -899,47 +895,21 @@ public class FunctionConfigUtils {
return retval;
}
- public static ClassLoader validate(FunctionConfig functionConfig, File functionPackageFile) {
+ public static void validateNonJavaFunction(FunctionConfig functionConfig) {
doCommonChecks(functionConfig);
- if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
- ClassLoader classLoader;
- if (functionPackageFile != null) {
- try {
- classLoader = loadJar(functionPackageFile);
- } catch (MalformedURLException e) {
- throw new IllegalArgumentException("Corrupted Jar File", e);
- }
- } else if (!isEmpty(functionConfig.getJar())) {
- File jarFile = new File(functionConfig.getJar());
- if (!jarFile.exists()) {
- throw new IllegalArgumentException("Jar file does not exist");
- }
- try {
- classLoader = loadJar(jarFile);
- } catch (Exception e) {
- throw new IllegalArgumentException("Corrupted Jar File", e);
- }
- } else {
- throw new IllegalArgumentException("Function Package is not provided");
- }
-
- doJavaChecks(functionConfig, classLoader);
- return classLoader;
- } else if (functionConfig.getRuntime() == FunctionConfig.Runtime.GO) {
+ if (functionConfig.getRuntime() == FunctionConfig.Runtime.GO) {
doGolangChecks(functionConfig);
- return null;
} else if (functionConfig.getRuntime() == FunctionConfig.Runtime.PYTHON) {
doPythonChecks(functionConfig);
- return null;
} else {
throw new IllegalArgumentException("Function language runtime is either not set or cannot be determined");
}
}
public static ExtractedFunctionDetails validateJavaFunction(FunctionConfig functionConfig,
- ClassLoader classLoader) {
+ ValidatableFunctionPackage validatableFunctionPackage) {
doCommonChecks(functionConfig);
- return doJavaChecks(functionConfig, classLoader);
+ return doJavaChecks(functionConfig, validatableFunctionPackage);
}
public static FunctionConfig validateUpdate(FunctionConfig existingConfig, FunctionConfig newConfig) {
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionFilePackage.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionFilePackage.java
new file mode 100644
index 00000000000..80b0bc05b3a
--- /dev/null
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionFilePackage.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.functions.utils;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.MalformedURLException;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.List;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.dynamic.ClassFileLocator;
+import net.bytebuddy.pool.TypePool;
+import org.apache.pulsar.common.nar.NarClassLoader;
+import org.apache.pulsar.common.nar.NarClassLoaderBuilder;
+import org.apache.pulsar.functions.utils.functions.FunctionUtils;
+import org.zeroturnaround.zip.ZipUtil;
+
+/**
+ * FunctionFilePackage is a class that represents a function package and
+ * implements the ValidatableFunctionPackage interface which decouples the
+ * function package from classloading.
+ */
+public class FunctionFilePackage implements AutoCloseable, ValidatableFunctionPackage {
+ private final File file;
+ private final ClassFileLocator.Compound classFileLocator;
+ private final TypePool typePool;
+ private final boolean isNar;
+ private final String narExtractionDirectory;
+ private final boolean enableClassloading;
+
+ private ClassLoader classLoader;
+
+ private final Object configMetadata;
+
+ public FunctionFilePackage(File file, String narExtractionDirectory, boolean enableClassloading,
+ Class<?> configClass) {
+ this.file = file;
+ boolean nonZeroFile = file.isFile() && file.length() > 0;
+ this.isNar = nonZeroFile ? ZipUtil.containsAnyEntry(file,
+ new String[] {"META-INF/services/pulsar-io.yaml", "META-INF/bundled-dependencies"}) : false;
+ this.narExtractionDirectory = narExtractionDirectory;
+ this.enableClassloading = enableClassloading;
+ if (isNar) {
+ List<File> classpathFromArchive = null;
+ try {
+ classpathFromArchive = NarClassLoader.getClasspathFromArchive(file, narExtractionDirectory);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ List<ClassFileLocator> classFileLocators = new ArrayList<>();
+ classFileLocators.add(ClassFileLocator.ForClassLoader.ofSystemLoader());
+ for (File classpath : classpathFromArchive) {
+ if (classpath.exists()) {
+ try {
+ ClassFileLocator locator;
+ if (classpath.isDirectory()) {
+ locator = new ClassFileLocator.ForFolder(classpath);
+ } else {
+ locator = ClassFileLocator.ForJarFile.of(classpath);
+ }
+ classFileLocators.add(locator);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+ }
+ this.classFileLocator = new ClassFileLocator.Compound(classFileLocators);
+ this.typePool = TypePool.Default.of(classFileLocator);
+ try {
+ this.configMetadata = FunctionUtils.getPulsarIOServiceConfig(file, configClass);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ } else {
+ try {
+ this.classFileLocator = nonZeroFile
+ ? new ClassFileLocator.Compound(ClassFileLocator.ForClassLoader.ofSystemLoader(),
+ ClassFileLocator.ForJarFile.of(file)) :
+ new ClassFileLocator.Compound(ClassFileLocator.ForClassLoader.ofSystemLoader());
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ this.typePool =
+ TypePool.Default.of(classFileLocator);
+ this.configMetadata = null;
+ }
+ }
+
+ public TypeDescription resolveType(String className) {
+ return typePool.describe(className).resolve();
+ }
+
+ public boolean isNar() {
+ return isNar;
+ }
+
+ public File getFile() {
+ return file;
+ }
+
+ public TypePool getTypePool() {
+ return typePool;
+ }
+
+ @Override
+ public <T> T getFunctionMetaData(Class<T> clazz) {
+ return configMetadata != null ? clazz.cast(configMetadata) : null;
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ classFileLocator.close();
+ if (classLoader instanceof Closeable) {
+ ((Closeable) classLoader).close();
+ }
+ }
+
+ public boolean isEnableClassloading() {
+ return enableClassloading;
+ }
+
+ public synchronized ClassLoader getClassLoader() {
+ if (classLoader == null) {
+ classLoader = createClassLoader();
+ }
+ return classLoader;
+ }
+
+ private ClassLoader createClassLoader() {
+ if (enableClassloading) {
+ if (isNar) {
+ try {
+ return NarClassLoaderBuilder.builder()
+ .narFile(file)
+ .extractionDirectory(narExtractionDirectory)
+ .build();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ } else {
+ try {
+ return new URLClassLoader(new java.net.URL[] {file.toURI().toURL()},
+ NarClassLoader.class.getClassLoader());
+ } catch (MalformedURLException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+ } else {
+ throw new IllegalStateException("Classloading is not enabled");
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "FunctionFilePackage{"
+ + "file=" + file
+ + ", isNar=" + isNar
+ + '}';
+ }
+}
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionRuntimeCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionRuntimeCommon.java
new file mode 100644
index 00000000000..948b1f1905a
--- /dev/null
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionRuntimeCommon.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.functions.utils;
+
+import static org.apache.commons.lang3.StringUtils.isEmpty;
+import java.io.File;
+import java.io.IOException;
+import org.apache.pulsar.common.nar.NarClassLoader;
+import org.apache.pulsar.common.nar.NarClassLoaderBuilder;
+import org.apache.pulsar.common.util.ClassLoaderUtils;
+import org.apache.pulsar.functions.proto.Function;
+import org.apache.pulsar.functions.utils.functions.FunctionUtils;
+import org.apache.pulsar.functions.utils.io.ConnectorUtils;
+
+public class FunctionRuntimeCommon {
+ public static NarClassLoader extractNarClassLoader(File packageFile,
+ String narExtractionDirectory) {
+ if (packageFile != null) {
+ try {
+ return NarClassLoaderBuilder.builder()
+ .narFile(packageFile)
+ .extractionDirectory(narExtractionDirectory)
+ .build();
+ } catch (IOException e) {
+ throw new IllegalArgumentException(e.getMessage());
+ }
+ }
+ return null;
+ }
+
+ public static ClassLoader getClassLoaderFromPackage(
+ Function.FunctionDetails.ComponentType componentType,
+ String className,
+ File packageFile,
+ String narExtractionDirectory) {
+ String connectorClassName = className;
+ ClassLoader jarClassLoader = null;
+ boolean keepJarClassLoader = false;
+ NarClassLoader narClassLoader = null;
+ boolean keepNarClassLoader = false;
+
+ Exception jarClassLoaderException = null;
+ Exception narClassLoaderException = null;
+
+ try {
+ try {
+ jarClassLoader = ClassLoaderUtils.extractClassLoader(packageFile);
+ } catch (Exception e) {
+ jarClassLoaderException = e;
+ }
+ try {
+ narClassLoader = extractNarClassLoader(packageFile, narExtractionDirectory);
+ } catch (Exception e) {
+ narClassLoaderException = e;
+ }
+
+ // if connector class name is not provided, we can only try to load archive as a NAR
+ if (isEmpty(connectorClassName)) {
+ if (narClassLoader == null) {
+ throw new IllegalArgumentException(String.format("%s package does not have the correct format. "
+ + "Pulsar cannot determine if the package is a NAR package or JAR package. "
+ + "%s classname is not provided and attempts to load it as a NAR package produced "
+ + "the following error.",
+ FunctionCommon.capFirstLetter(componentType), FunctionCommon.capFirstLetter(componentType)),
+ narClassLoaderException);
+ }
+ try {
+ if (componentType == Function.FunctionDetails.ComponentType.FUNCTION) {
+ connectorClassName = FunctionUtils.getFunctionClass(narClassLoader);
+ } else if (componentType == Function.FunctionDetails.ComponentType.SOURCE) {
+ connectorClassName = ConnectorUtils.getIOSourceClass(narClassLoader);
+ } else {
+ connectorClassName = ConnectorUtils.getIOSinkClass(narClassLoader);
+ }
+ } catch (IOException e) {
+ throw new IllegalArgumentException(String.format("Failed to extract %s class from archive",
+ componentType.toString().toLowerCase()), e);
+ }
+
+ try {
+ narClassLoader.loadClass(connectorClassName);
+ keepNarClassLoader = true;
+ return narClassLoader;
+ } catch (ClassNotFoundException | NoClassDefFoundError e) {
+ throw new IllegalArgumentException(String.format("%s class %s must be in class path",
+ FunctionCommon.capFirstLetter(componentType), connectorClassName), e);
+ }
+
+ } else {
+ // if connector class name is provided, we need to try to load it as a JAR and as a NAR.
+ if (jarClassLoader != null) {
+ try {
+ jarClassLoader.loadClass(connectorClassName);
+ keepJarClassLoader = true;
+ return jarClassLoader;
+ } catch (ClassNotFoundException | NoClassDefFoundError e) {
+ // class not found in JAR try loading as a NAR and searching for the class
+ if (narClassLoader != null) {
+
+ try {
+ narClassLoader.loadClass(connectorClassName);
+ keepNarClassLoader = true;
+ return narClassLoader;
+ } catch (ClassNotFoundException | NoClassDefFoundError e1) {
+ throw new IllegalArgumentException(
+ String.format("%s class %s must be in class path",
+ FunctionCommon.capFirstLetter(componentType), connectorClassName), e1);
+ }
+ } else {
+ throw new IllegalArgumentException(String.format("%s class %s must be in class path",
+ FunctionCommon.capFirstLetter(componentType), connectorClassName), e);
+ }
+ }
+ } else if (narClassLoader != null) {
+ try {
+ narClassLoader.loadClass(connectorClassName);
+ keepNarClassLoader = true;
+ return narClassLoader;
+ } catch (ClassNotFoundException | NoClassDefFoundError e1) {
+ throw new IllegalArgumentException(
+ String.format("%s class %s must be in class path",
+ FunctionCommon.capFirstLetter(componentType), connectorClassName), e1);
+ }
+ } else {
+ StringBuilder errorMsg = new StringBuilder(FunctionCommon.capFirstLetter(componentType)
+ + " package does not have the correct format."
+ + " Pulsar cannot determine if the package is a NAR package or JAR package.");
+
+ if (jarClassLoaderException != null) {
+ errorMsg.append(
+ " Attempts to load it as a JAR package produced error: " + jarClassLoaderException
+ .getMessage());
+ }
+
+ if (narClassLoaderException != null) {
+ errorMsg.append(
+ " Attempts to load it as a NAR package produced error: " + narClassLoaderException
+ .getMessage());
+ }
+
+ throw new IllegalArgumentException(errorMsg.toString());
+ }
+ }
+ } finally {
+ if (!keepJarClassLoader) {
+ ClassLoaderUtils.closeClassLoader(jarClassLoader);
+ }
+ if (!keepNarClassLoader) {
+ ClassLoaderUtils.closeClassLoader(narClassLoader);
+ }
+ }
+ }
+
+}
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/LoadedFunctionPackage.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/LoadedFunctionPackage.java
new file mode 100644
index 00000000000..3f885a7cd2a
--- /dev/null
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/LoadedFunctionPackage.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.functions.utils;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.dynamic.ClassFileLocator;
+import net.bytebuddy.pool.TypePool;
+import org.apache.pulsar.common.nar.NarClassLoader;
+import org.apache.pulsar.functions.utils.functions.FunctionUtils;
+
+/**
+ * LoadedFunctionPackage is a class that represents a function package and
+ * implements the ValidatableFunctionPackage interface which decouples the
+ * function package from classloading. This implementation is backed by
+ * a ClassLoader, and it is used when the function package is already loaded
+ * by a ClassLoader. This is the case in the LocalRunner and in some of
+ * the unit tests.
+ */
+public class LoadedFunctionPackage implements ValidatableFunctionPackage {
+ private final ClassLoader classLoader;
+ private final Object configMetadata;
+ private final TypePool typePool;
+
+ public <T> LoadedFunctionPackage(ClassLoader classLoader, Class<T> configMetadataClass, T configMetadata) {
+ this.classLoader = classLoader;
+ this.configMetadata = configMetadata;
+ typePool = TypePool.Default.of(
+ ClassFileLocator.ForClassLoader.of(classLoader));
+ }
+
+ public LoadedFunctionPackage(ClassLoader classLoader, Class<?> configMetadataClass) {
+ this.classLoader = classLoader;
+ if (classLoader instanceof NarClassLoader) {
+ try {
+ configMetadata = FunctionUtils.getPulsarIOServiceConfig((NarClassLoader) classLoader,
+ configMetadataClass);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ } else {
+ configMetadata = null;
+ }
+ typePool = TypePool.Default.of(
+ ClassFileLocator.ForClassLoader.of(classLoader));
+ }
+
+ @Override
+ public TypeDescription resolveType(String className) {
+ return typePool.describe(className).resolve();
+ }
+
+ @Override
+ public TypePool getTypePool() {
+ return typePool;
+ }
+
+ @Override
+ public <T> T getFunctionMetaData(Class<T> clazz) {
+ return configMetadata != null ? clazz.cast(configMetadata) : null;
+ }
+
+ @Override
+ public boolean isEnableClassloading() {
+ return true;
+ }
+
+ @Override
+ public ClassLoader getClassLoader() {
+ return classLoader;
+ }
+}
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java
index 34574cc4d44..f4b7bf25134 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java
@@ -39,6 +39,8 @@ import lombok.Getter;
import lombok.Setter;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
+import net.bytebuddy.description.type.TypeDefinition;
+import net.bytebuddy.pool.TypePool;
import org.apache.commons.lang.StringUtils;
import org.apache.pulsar.client.api.SubscriptionInitialPosition;
import org.apache.pulsar.common.functions.ConsumerConfig;
@@ -47,13 +49,11 @@ import org.apache.pulsar.common.functions.Resources;
import org.apache.pulsar.common.io.ConnectorDefinition;
import org.apache.pulsar.common.io.SinkConfig;
import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.nar.NarClassLoader;
import org.apache.pulsar.common.util.ObjectMapperFactory;
import org.apache.pulsar.config.validation.ConfigValidation;
import org.apache.pulsar.functions.api.utils.IdentityFunction;
import org.apache.pulsar.functions.proto.Function;
import org.apache.pulsar.functions.proto.Function.FunctionDetails;
-import org.apache.pulsar.functions.utils.io.ConnectorUtils;
@Slf4j
public class SinkConfigUtils {
@@ -190,12 +190,6 @@ public class SinkConfigUtils {
sourceSpecBuilder.setNegativeAckRedeliveryDelayMs(sinkConfig.getNegativeAckRedeliveryDelayMs());
}
- if (sinkConfig.getCleanupSubscription() != null) {
- sourceSpecBuilder.setCleanupSubscription(sinkConfig.getCleanupSubscription());
- } else {
- sourceSpecBuilder.setCleanupSubscription(true);
- }
-
if (sinkConfig.getSourceSubscriptionPosition() == SubscriptionInitialPosition.Earliest) {
sourceSpecBuilder.setSubscriptionPosition(Function.SubscriptionPosition.EARLIEST);
} else {
@@ -319,7 +313,6 @@ public class SinkConfigUtils {
// Set subscription position
sinkConfig.setSourceSubscriptionPosition(
convertFromFunctionDetailsSubscriptionPosition(functionDetails.getSource().getSubscriptionPosition()));
- sinkConfig.setCleanupSubscription(functionDetails.getSource().getCleanupSubscription());
if (functionDetails.getSource().getTimeoutMs() != 0) {
sinkConfig.setTimeoutMs(functionDetails.getSource().getTimeoutMs());
@@ -380,7 +373,7 @@ public class SinkConfigUtils {
}
public static ExtractedSinkDetails validateAndExtractDetails(SinkConfig sinkConfig,
- ClassLoader sinkClassLoader,
+ ValidatableFunctionPackage sinkFunction,
boolean validateConnectorConfig) {
if (isEmpty(sinkConfig.getTenant())) {
throw new IllegalArgumentException("Sink tenant cannot be null");
@@ -420,34 +413,38 @@ public class SinkConfigUtils {
// if class name in sink config is not set, this should be a built-in sink
// thus we should try to find it class name in the NAR service definition
if (sinkClassName == null) {
- try {
- sinkClassName = ConnectorUtils.getIOSinkClass((NarClassLoader) sinkClassLoader);
- } catch (IOException e) {
- throw new IllegalArgumentException("Failed to extract sink class from archive", e);
+ ConnectorDefinition connectorDefinition = sinkFunction.getFunctionMetaData(ConnectorDefinition.class);
+ if (connectorDefinition == null) {
+ throw new IllegalArgumentException(
+ "Sink package doesn't contain the META-INF/services/pulsar-io.yaml file.");
+ }
+ sinkClassName = connectorDefinition.getSinkClass();
+ if (sinkClassName == null) {
+ throw new IllegalArgumentException("Failed to extract sink class from archive");
}
}
// check if sink implements the correct interfaces
- Class sinkClass;
+ TypeDefinition sinkClass;
try {
- sinkClass = sinkClassLoader.loadClass(sinkClassName);
- } catch (ClassNotFoundException e) {
+ sinkClass = sinkFunction.resolveType(sinkClassName);
+ } catch (TypePool.Resolution.NoSuchTypeException e) {
throw new IllegalArgumentException(
- String.format("Sink class %s not found in class loader", sinkClassName), e);
+ String.format("Sink class %s not found", sinkClassName), e);
}
- // extract type from sink class
- Class<?> typeArg = getSinkType(sinkClass);
+ TypeDefinition typeArg = getSinkType(sinkClass);
+ ValidatableFunctionPackage inputFunction = sinkFunction;
if (sinkConfig.getTopicToSerdeClassName() != null) {
for (String serdeClassName : sinkConfig.getTopicToSerdeClassName().values()) {
- ValidatorUtils.validateSerde(serdeClassName, typeArg, sinkClassLoader, true);
+ ValidatorUtils.validateSerde(serdeClassName, typeArg, inputFunction.getTypePool(), true);
}
}
if (sinkConfig.getTopicToSchemaType() != null) {
for (String schemaType : sinkConfig.getTopicToSchemaType().values()) {
- ValidatorUtils.validateSchema(schemaType, typeArg, sinkClassLoader, true);
+ ValidatorUtils.validateSchema(schemaType, typeArg, inputFunction.getTypePool(), true);
}
}
@@ -460,26 +457,46 @@ public class SinkConfigUtils {
throw new IllegalArgumentException("Only one of serdeClassName or schemaType should be set");
}
if (!isEmpty(consumerSpec.getSerdeClassName())) {
- ValidatorUtils.validateSerde(consumerSpec.getSerdeClassName(), typeArg, sinkClassLoader, true);
+ ValidatorUtils.validateSerde(consumerSpec.getSerdeClassName(), typeArg,
+ inputFunction.getTypePool(), true);
}
if (!isEmpty(consumerSpec.getSchemaType())) {
- ValidatorUtils.validateSchema(consumerSpec.getSchemaType(), typeArg, sinkClassLoader, true);
+ ValidatorUtils.validateSchema(consumerSpec.getSchemaType(), typeArg,
+ inputFunction.getTypePool(), true);
}
if (consumerSpec.getCryptoConfig() != null) {
- ValidatorUtils.validateCryptoKeyReader(consumerSpec.getCryptoConfig(), sinkClassLoader, false);
+ ValidatorUtils.validateCryptoKeyReader(consumerSpec.getCryptoConfig(),
+ inputFunction.getTypePool(), false);
}
}
}
- // validate user defined config if enabled and sink is loaded from NAR
- if (validateConnectorConfig && sinkClassLoader instanceof NarClassLoader) {
- validateSinkConfig(sinkConfig, (NarClassLoader) sinkClassLoader);
+ if (sinkConfig.getRetainKeyOrdering() != null
+ && sinkConfig.getRetainKeyOrdering()
+ && sinkConfig.getProcessingGuarantees() != null
+ && sinkConfig.getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) {
+ throw new IllegalArgumentException(
+ "When effectively once processing guarantee is specified, retain Key ordering cannot be set");
+ }
+
+ if (sinkConfig.getRetainKeyOrdering() != null && sinkConfig.getRetainKeyOrdering()
+ && sinkConfig.getRetainOrdering() != null && sinkConfig.getRetainOrdering()) {
+ throw new IllegalArgumentException("Only one of retain ordering or retain key ordering can be set");
}
- return new ExtractedSinkDetails(sinkClassName, typeArg.getName());
+ // validate user defined config if enabled and classloading is enabled
+ if (validateConnectorConfig) {
+ if (sinkFunction.isEnableClassloading()) {
+ validateSinkConfig(sinkConfig, sinkFunction);
+ } else {
+ log.warn("Skipping annotation based validation of sink config as classloading is disabled");
+ }
+ }
+
+ return new ExtractedSinkDetails(sinkClassName, typeArg.asErasure().getTypeName());
}
- private static Collection<String> collectAllInputTopics(SinkConfig sinkConfig) {
+ public static Collection<String> collectAllInputTopics(SinkConfig sinkConfig) {
List<String> retval = new LinkedList<>();
if (sinkConfig.getInputs() != null) {
retval.addAll(sinkConfig.getInputs());
@@ -501,8 +518,8 @@ public class SinkConfigUtils {
@SneakyThrows
public static SinkConfig clone(SinkConfig sinkConfig) {
- return ObjectMapperFactory.getThreadLocal().readValue(
- ObjectMapperFactory.getThreadLocal().writeValueAsBytes(sinkConfig), SinkConfig.class);
+ return ObjectMapperFactory.getThreadLocal().reader().readValue(
+ ObjectMapperFactory.getThreadLocal().writer().writeValueAsBytes(sinkConfig), SinkConfig.class);
}
public static SinkConfig validateUpdate(SinkConfig existingConfig, SinkConfig newConfig) {
@@ -535,7 +552,7 @@ public class SinkConfigUtils {
if (newConfig.getInputs() != null) {
newConfig.getInputs().forEach((topicName -> {
- newConfig.getInputSpecs().put(topicName,
+ newConfig.getInputSpecs().putIfAbsent(topicName,
ConsumerConfig.builder().isRegexPattern(false).build());
}));
}
@@ -619,36 +636,17 @@ public class SinkConfigUtils {
if (!StringUtils.isEmpty(newConfig.getCustomRuntimeOptions())) {
mergedConfig.setCustomRuntimeOptions(newConfig.getCustomRuntimeOptions());
}
- if (newConfig.getCleanupSubscription() != null) {
- mergedConfig.setCleanupSubscription(newConfig.getCleanupSubscription());
- }
return mergedConfig;
}
- public static void validateSinkConfig(SinkConfig sinkConfig, NarClassLoader narClassLoader) {
-
- if (sinkConfig.getRetainKeyOrdering() != null
- && sinkConfig.getRetainKeyOrdering()
- && sinkConfig.getProcessingGuarantees() != null
- && sinkConfig.getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) {
- throw new IllegalArgumentException(
- "When effectively once processing guarantee is specified, retain Key ordering cannot be set");
- }
-
- if (sinkConfig.getRetainKeyOrdering() != null && sinkConfig.getRetainKeyOrdering()
- && sinkConfig.getRetainOrdering() != null && sinkConfig.getRetainOrdering()) {
- throw new IllegalArgumentException("Only one of retain ordering or retain key ordering can be set");
- }
-
+ public static void validateSinkConfig(SinkConfig sinkConfig, ValidatableFunctionPackage sinkFunction) {
try {
- ConnectorDefinition defn = ConnectorUtils.getConnectorDefinition(narClassLoader);
- if (defn.getSinkConfigClass() != null) {
- Class configClass = Class.forName(defn.getSinkConfigClass(), true, narClassLoader);
+ ConnectorDefinition defn = sinkFunction.getFunctionMetaData(ConnectorDefinition.class);
+ if (defn != null && defn.getSinkConfigClass() != null) {
+ Class configClass = Class.forName(defn.getSinkConfigClass(), true, sinkFunction.getClassLoader());
validateSinkConfig(sinkConfig, configClass);
}
- } catch (IOException e) {
- throw new IllegalArgumentException("Error validating sink config", e);
} catch (ClassNotFoundException e) {
throw new IllegalArgumentException("Could not find sink config class", e);
}
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java
index dd32903f4e4..207d426ab54 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java
@@ -34,7 +34,9 @@ import lombok.Getter;
import lombok.Setter;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
-import net.jodah.typetools.TypeResolver;
+import net.bytebuddy.description.type.TypeDefinition;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.pool.TypePool;
import org.apache.commons.lang3.StringUtils;
import org.apache.pulsar.common.functions.ProducerConfig;
import org.apache.pulsar.common.functions.Resources;
@@ -43,13 +45,11 @@ import org.apache.pulsar.common.io.ConnectorDefinition;
import org.apache.pulsar.common.io.SourceConfig;
import org.apache.pulsar.common.naming.TopicDomain;
import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.nar.NarClassLoader;
import org.apache.pulsar.common.util.ObjectMapperFactory;
import org.apache.pulsar.config.validation.ConfigValidation;
import org.apache.pulsar.functions.api.utils.IdentityFunction;
import org.apache.pulsar.functions.proto.Function;
import org.apache.pulsar.functions.proto.Function.FunctionDetails;
-import org.apache.pulsar.functions.utils.io.ConnectorUtils;
import org.apache.pulsar.io.core.BatchSource;
import org.apache.pulsar.io.core.Source;
@@ -285,7 +285,7 @@ public class SourceConfigUtils {
}
public static ExtractedSourceDetails validateAndExtractDetails(SourceConfig sourceConfig,
- ClassLoader sourceClassLoader,
+ ValidatableFunctionPackage sourceFunction,
boolean validateConnectorConfig) {
if (isEmpty(sourceConfig.getTenant())) {
throw new IllegalArgumentException("Source tenant cannot be null");
@@ -313,29 +313,34 @@ public class SourceConfigUtils {
// if class name in source config is not set, this should be a built-in source
// thus we should try to find it class name in the NAR service definition
if (sourceClassName == null) {
- try {
- sourceClassName = ConnectorUtils.getIOSourceClass((NarClassLoader) sourceClassLoader);
- } catch (IOException e) {
- throw new IllegalArgumentException("Failed to extract source class from archive", e);
+ ConnectorDefinition connectorDefinition = sourceFunction.getFunctionMetaData(ConnectorDefinition.class);
+ if (connectorDefinition == null) {
+ throw new IllegalArgumentException(
+ "Source package doesn't contain the META-INF/services/pulsar-io.yaml file.");
+ }
+ sourceClassName = connectorDefinition.getSourceClass();
+ if (sourceClassName == null) {
+ throw new IllegalArgumentException("Failed to extract source class from archive");
}
}
// check if source implements the correct interfaces
- Class sourceClass;
+ TypeDescription sourceClass;
try {
- sourceClass = sourceClassLoader.loadClass(sourceClassName);
- } catch (ClassNotFoundException e) {
+ sourceClass = sourceFunction.resolveType(sourceClassName);
+ } catch (TypePool.Resolution.NoSuchTypeException e) {
throw new IllegalArgumentException(
String.format("Source class %s not found in class loader", sourceClassName), e);
}
- if (!Source.class.isAssignableFrom(sourceClass) && !BatchSource.class.isAssignableFrom(sourceClass)) {
+ if (!(sourceClass.asErasure().isAssignableTo(Source.class) || sourceClass.asErasure()
+ .isAssignableTo(BatchSource.class))) {
throw new IllegalArgumentException(
- String.format("Source class %s does not implement the correct interface",
- sourceClass.getName()));
+ String.format("Source class %s does not implement the correct interface",
+ sourceClass.getName()));
}
- if (BatchSource.class.isAssignableFrom(sourceClass)) {
+ if (sourceClass.asErasure().isAssignableTo(BatchSource.class)) {
if (sourceConfig.getBatchSourceConfig() != null) {
validateBatchSourceConfig(sourceConfig.getBatchSourceConfig());
} else {
@@ -346,7 +351,14 @@ public class SourceConfigUtils {
}
// extract type from source class
- Class<?> typeArg = getSourceType(sourceClass);
+ TypeDefinition typeArg;
+
+ try {
+ typeArg = getSourceType(sourceClass);
+ } catch (Exception e) {
+ throw new IllegalArgumentException(
+ String.format("Failed to resolve type for Source class %s", sourceClassName), e);
+ }
// Only one of serdeClassName or schemaType should be set
if (!StringUtils.isEmpty(sourceConfig.getSerdeClassName()) && !StringUtils
@@ -355,29 +367,30 @@ public class SourceConfigUtils {
}
if (!StringUtils.isEmpty(sourceConfig.getSerdeClassName())) {
- ValidatorUtils.validateSerde(sourceConfig.getSerdeClassName(), typeArg, sourceClassLoader, false);
+ ValidatorUtils.validateSerde(sourceConfig.getSerdeClassName(), typeArg, sourceFunction.getTypePool(),
+ false);
}
if (!StringUtils.isEmpty(sourceConfig.getSchemaType())) {
- ValidatorUtils.validateSchema(sourceConfig.getSchemaType(), typeArg, sourceClassLoader, false);
+ ValidatorUtils.validateSchema(sourceConfig.getSchemaType(), typeArg, sourceFunction.getTypePool(),
+ false);
}
if (sourceConfig.getProducerConfig() != null && sourceConfig.getProducerConfig().getCryptoConfig() != null) {
ValidatorUtils
- .validateCryptoKeyReader(sourceConfig.getProducerConfig().getCryptoConfig(), sourceClassLoader,
- true);
+ .validateCryptoKeyReader(sourceConfig.getProducerConfig().getCryptoConfig(),
+ sourceFunction.getTypePool(), true);
}
- if (typeArg.equals(TypeResolver.Unknown.class)) {
- throw new IllegalArgumentException(
- String.format("Failed to resolve type for Source class %s", sourceClassName));
- }
-
- // validate user defined config if enabled and source is loaded from NAR
- if (validateConnectorConfig && sourceClassLoader instanceof NarClassLoader) {
- validateSourceConfig(sourceConfig, (NarClassLoader) sourceClassLoader);
+ // validate user defined config if enabled and classloading is enabled
+ if (validateConnectorConfig) {
+ if (sourceFunction.isEnableClassloading()) {
+ validateSourceConfig(sourceConfig, sourceFunction);
+ } else {
+ log.warn("Skipping annotation based validation of sink config as classloading is disabled");
+ }
}
- return new ExtractedSourceDetails(sourceClassName, typeArg.getName());
+ return new ExtractedSourceDetails(sourceClassName, typeArg.asErasure().getTypeName());
}
@SneakyThrows
@@ -518,15 +531,14 @@ public class SourceConfigUtils {
}
}
- public static void validateSourceConfig(SourceConfig sourceConfig, NarClassLoader narClassLoader) {
+ public static void validateSourceConfig(SourceConfig sourceConfig, ValidatableFunctionPackage sourceFunction) {
try {
- ConnectorDefinition defn = ConnectorUtils.getConnectorDefinition(narClassLoader);
- if (defn.getSourceConfigClass() != null) {
- Class configClass = Class.forName(defn.getSourceConfigClass(), true, narClassLoader);
+ ConnectorDefinition defn = sourceFunction.getFunctionMetaData(ConnectorDefinition.class);
+ if (defn != null && defn.getSourceConfigClass() != null) {
+ Class configClass =
+ Class.forName(defn.getSourceConfigClass(), true, sourceFunction.getClassLoader());
validateSourceConfig(sourceConfig, configClass);
}
- } catch (IOException e) {
- throw new IllegalArgumentException("Error validating source config", e);
} catch (ClassNotFoundException e) {
throw new IllegalArgumentException("Could not find source config class");
}
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatableFunctionPackage.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatableFunctionPackage.java
new file mode 100644
index 00000000000..bd71bbfb5a5
--- /dev/null
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatableFunctionPackage.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.functions.utils;
+
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.pool.TypePool;
+
+/**
+ * This abstraction separates the function and connector definition from classloading,
+ * enabling validation without the need for classloading. It utilizes Byte Buddy for
+ * type and annotation resolution.
+ *
+ * The function or connector definition is directly extracted from the archive file,
+ * eliminating the need for classloader initialization.
+ *
+ * The getClassLoader method should only be invoked when classloading is enabled.
+ * Classloading is required in the LocalRunner and in the Functions worker when the
+ * worker is configured with the 'validateConnectorConfig' set to true.
+ */
+public interface ValidatableFunctionPackage {
+ /**
+ * Resolves the type description for the given class name within the function package.
+ */
+ TypeDescription resolveType(String className);
+ /**
+ * Returns the Byte Buddy TypePool instance for the function package.
+ */
+ TypePool getTypePool();
+ /**
+ * Returns the function or connector definition metadata.
+ * Supports FunctionDefinition and ConnectorDefinition as the metadata type.
+ */
+ <T> T getFunctionMetaData(Class<T> clazz);
+ /**
+ * Returns if classloading is enabled for the function package.
+ */
+ boolean isEnableClassloading();
+ /**
+ * Returns the classloader for the function package. The classloader is
+ * lazily initialized when classloading is enabled.
+ */
+ ClassLoader getClassLoader();
+}
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatorUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatorUtils.java
index 9e9ee27dfeb..1d12deaaea5 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatorUtils.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatorUtils.java
@@ -18,35 +18,40 @@
*/
package org.apache.pulsar.functions.utils;
-import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isEmpty;
-import static org.apache.commons.lang3.StringUtils.isNotBlank;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
-import net.jodah.typetools.TypeResolver;
+import net.bytebuddy.description.type.TypeDefinition;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.pool.TypePool;
import org.apache.pulsar.client.api.CryptoKeyReader;
import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.common.functions.CryptoConfig;
import org.apache.pulsar.common.schema.SchemaType;
-import org.apache.pulsar.common.util.ClassLoaderUtils;
-import org.apache.pulsar.common.util.Reflections;
import org.apache.pulsar.functions.api.SerDe;
-import org.apache.pulsar.functions.proto.Function;
-import org.apache.pulsar.io.core.Sink;
-import org.apache.pulsar.io.core.Source;
@Slf4j
public class ValidatorUtils {
private static final String DEFAULT_SERDE = "org.apache.pulsar.functions.api.utils.DefaultSerDe";
- public static void validateSchema(String schemaType, Class<?> typeArg, ClassLoader clsLoader,
+ public static void validateSchema(String schemaType, TypeDefinition typeArg, TypePool typePool,
boolean input) {
if (isEmpty(schemaType) || getBuiltinSchemaType(schemaType) != null) {
// If it's empty, we use the default schema and no need to validate
// If it's built-in, no need to validate
} else {
- ClassLoaderUtils.implementsClass(schemaType, Schema.class, clsLoader);
- validateSchemaType(schemaType, typeArg, clsLoader, input);
+ TypeDescription schemaClass = null;
+ try {
+ schemaClass = typePool.describe(schemaType).resolve();
+ } catch (TypePool.Resolution.NoSuchTypeException e) {
+ throw new IllegalArgumentException(
+ String.format("The schema class %s does not exist", schemaType));
+ }
+ if (!schemaClass.asErasure().isAssignableTo(Schema.class)) {
+ throw new IllegalArgumentException(
+ String.format("%s does not implement %s", schemaType, Schema.class.getName()));
+ }
+ validateSchemaType(schemaClass, typeArg, typePool, input);
}
}
@@ -60,29 +65,32 @@ public class ValidatorUtils {
}
- public static void validateCryptoKeyReader(CryptoConfig conf, ClassLoader classLoader, boolean isProducer) {
+ public static void validateCryptoKeyReader(CryptoConfig conf, TypePool typePool, boolean isProducer) {
if (isEmpty(conf.getCryptoKeyReaderClassName())) {
return;
}
- Class<?> cryptoClass;
+ String cryptoClassName = conf.getCryptoKeyReaderClassName();
+ TypeDescription cryptoClass = null;
try {
- cryptoClass = ClassLoaderUtils.loadClass(conf.getCryptoKeyReaderClassName(), classLoader);
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
+ cryptoClass = typePool.describe(cryptoClassName).resolve();
+ } catch (TypePool.Resolution.NoSuchTypeException e) {
throw new IllegalArgumentException(
- String.format("The crypto key reader class %s does not exist", conf.getCryptoKeyReaderClassName()));
+ String.format("The crypto key reader class %s does not exist", cryptoClassName));
+ }
+ if (!cryptoClass.asErasure().isAssignableTo(CryptoKeyReader.class)) {
+ throw new IllegalArgumentException(
+ String.format("%s does not implement %s", cryptoClassName, CryptoKeyReader.class.getName()));
}
- ClassLoaderUtils.implementsClass(conf.getCryptoKeyReaderClassName(), CryptoKeyReader.class, classLoader);
- try {
- cryptoClass.getConstructor(Map.class);
- } catch (NoSuchMethodException ex) {
+ boolean hasConstructor = cryptoClass.getDeclaredMethods().stream()
+ .anyMatch(method -> method.isConstructor() && method.getParameters().size() == 1
+ && method.getParameters().get(0).getType().asErasure().represents(Map.class));
+
+ if (!hasConstructor) {
throw new IllegalArgumentException(
String.format("The crypto key reader class %s does not implement the desired constructor.",
conf.getCryptoKeyReaderClassName()));
-
- } catch (SecurityException e) {
- throw new IllegalArgumentException("Failed to access crypto key reader class", e);
}
if (isProducer && (conf.getEncryptionKeys() == null || conf.getEncryptionKeys().length == 0)) {
@@ -90,7 +98,7 @@ public class ValidatorUtils {
}
}
- public static void validateSerde(String inputSerializer, Class<?> typeArg, ClassLoader clsLoader,
+ public static void validateSerde(String inputSerializer, TypeDefinition typeArg, TypePool typePool,
boolean deser) {
if (isEmpty(inputSerializer)) {
return;
@@ -98,154 +106,53 @@ public class ValidatorUtils {
if (inputSerializer.equals(DEFAULT_SERDE)) {
return;
}
+ TypeDescription serdeClass;
try {
- Class<?> serdeClass = ClassLoaderUtils.loadClass(inputSerializer, clsLoader);
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
+ serdeClass = typePool.describe(inputSerializer).resolve();
+ } catch (TypePool.Resolution.NoSuchTypeException e) {
throw new IllegalArgumentException(
String.format("The input serialization/deserialization class %s does not exist",
inputSerializer));
}
- ClassLoaderUtils.implementsClass(inputSerializer, SerDe.class, clsLoader);
-
- SerDe serDe = (SerDe) Reflections.createInstance(inputSerializer, clsLoader);
- if (serDe == null) {
- throw new IllegalArgumentException(String.format("The SerDe class %s does not exist",
- inputSerializer));
- }
- Class<?>[] serDeTypes = TypeResolver.resolveRawArguments(SerDe.class, serDe.getClass());
-
- // type inheritance information seems to be lost in generic type
- // load the actual type class for verification
- Class<?> fnInputClass;
- Class<?> serdeInputClass;
- try {
- fnInputClass = Class.forName(typeArg.getName(), true, clsLoader);
- serdeInputClass = Class.forName(serDeTypes[0].getName(), true, clsLoader);
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
- throw new IllegalArgumentException("Failed to load type class", e);
- }
+ TypeDescription.Generic serDeTypeArg = serdeClass.getInterfaces().stream()
+ .filter(i -> i.asErasure().isAssignableTo(SerDe.class))
+ .findFirst()
+ .map(i -> i.getTypeArguments().get(0))
+ .orElseThrow(() -> new IllegalArgumentException(
+ String.format("%s does not implement %s", inputSerializer, SerDe.class.getName())));
if (deser) {
- if (!fnInputClass.isAssignableFrom(serdeInputClass)) {
- throw new IllegalArgumentException("Serializer type mismatch " + typeArg + " vs " + serDeTypes[0]);
+ if (!serDeTypeArg.asErasure().isAssignableTo(typeArg.asErasure())) {
+ throw new IllegalArgumentException("Serializer type mismatch " + typeArg.getActualName() + " vs "
+ + serDeTypeArg.getActualName());
}
} else {
- if (!serdeInputClass.isAssignableFrom(fnInputClass)) {
- throw new IllegalArgumentException("Serializer type mismatch " + typeArg + " vs " + serDeTypes[0]);
+ if (!serDeTypeArg.asErasure().isAssignableFrom(typeArg.asErasure())) {
+ throw new IllegalArgumentException("Serializer type mismatch " + typeArg.getActualName() + " vs "
+ + serDeTypeArg.getActualName());
}
}
}
- private static void validateSchemaType(String schemaClassName, Class<?> typeArg, ClassLoader clsLoader,
+ private static void validateSchemaType(TypeDefinition schema, TypeDefinition typeArg, TypePool typePool,
boolean input) {
- Schema<?> schema = (Schema<?>) Reflections.createInstance(schemaClassName, clsLoader);
- if (schema == null) {
- throw new IllegalArgumentException(String.format("The Schema class %s does not exist",
- schemaClassName));
- }
- Class<?>[] schemaTypes = TypeResolver.resolveRawArguments(Schema.class, schema.getClass());
- // type inheritance information seems to be lost in generic type
- // load the actual type class for verification
- Class<?> fnInputClass;
- Class<?> schemaInputClass;
- try {
- fnInputClass = Class.forName(typeArg.getName(), true, clsLoader);
- schemaInputClass = Class.forName(schemaTypes[0].getName(), true, clsLoader);
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
- throw new IllegalArgumentException("Failed to load type class", e);
- }
+ TypeDescription.Generic schemaTypeArg = schema.getInterfaces().stream()
+ .filter(i -> i.asErasure().isAssignableTo(Schema.class))
+ .findFirst()
+ .map(i -> i.getTypeArguments().get(0))
+ .orElse(null);
if (input) {
- if (!fnInputClass.isAssignableFrom(schemaInputClass)) {
+ if (!schemaTypeArg.asErasure().isAssignableTo(typeArg.asErasure())) {
throw new IllegalArgumentException(
- "Schema type mismatch " + typeArg + " vs " + schemaTypes[0]);
+ "Schema type mismatch " + typeArg.getActualName() + " vs " + schemaTypeArg.getActualName());
}
} else {
- if (!schemaInputClass.isAssignableFrom(fnInputClass)) {
+ if (!schemaTypeArg.asErasure().isAssignableFrom(typeArg.asErasure())) {
throw new IllegalArgumentException(
- "Schema type mismatch " + typeArg + " vs " + schemaTypes[0]);
- }
- }
- }
-
-
- public static void validateFunctionClassTypes(ClassLoader classLoader,
- Function.FunctionDetails.Builder functionDetailsBuilder) {
-
- // validate only if classLoader is provided
- if (classLoader == null) {
- return;
- }
-
- if (isBlank(functionDetailsBuilder.getClassName())) {
- throw new IllegalArgumentException("Function class-name can't be empty");
- }
-
- // validate function class-type
- Class functionClass;
- try {
- functionClass = classLoader.loadClass(functionDetailsBuilder.getClassName());
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
- throw new IllegalArgumentException(
- String.format("Function class %s must be in class path", functionDetailsBuilder.getClassName()), e);
- }
- Class<?>[] typeArgs = FunctionCommon.getFunctionTypes(functionClass, false);
-
- if (!(org.apache.pulsar.functions.api.Function.class.isAssignableFrom(functionClass))
- && !(java.util.function.Function.class.isAssignableFrom(functionClass))) {
- throw new RuntimeException("User class must either be Function or java.util.Function");
- }
-
- if (functionDetailsBuilder.hasSource() && functionDetailsBuilder.getSource() != null
- && isNotBlank(functionDetailsBuilder.getSource().getClassName())) {
- try {
- String sourceClassName = functionDetailsBuilder.getSource().getClassName();
- String argClassName = FunctionCommon.getTypeArg(sourceClassName, Source.class, classLoader).getName();
- functionDetailsBuilder
- .setSource(functionDetailsBuilder.getSourceBuilder().setTypeClassName(argClassName));
-
- // if sink-class not present then set same arg as source
- if (!functionDetailsBuilder.hasSink() || isBlank(functionDetailsBuilder.getSink().getClassName())) {
- functionDetailsBuilder
- .setSink(functionDetailsBuilder.getSinkBuilder().setTypeClassName(argClassName));
- }
-
- } catch (IllegalArgumentException ie) {
- throw ie;
- } catch (Exception e) {
- log.error("Failed to validate source class", e);
- throw new IllegalArgumentException("Failed to validate source class-name", e);
- }
- } else if (isBlank(functionDetailsBuilder.getSourceBuilder().getTypeClassName())) {
- // if function-src-class is not present then set function-src type-class according to function class
- functionDetailsBuilder
- .setSource(functionDetailsBuilder.getSourceBuilder().setTypeClassName(typeArgs[0].getName()));
- }
-
- if (functionDetailsBuilder.hasSink() && functionDetailsBuilder.getSink() != null
- && isNotBlank(functionDetailsBuilder.getSink().getClassName())) {
- try {
- String sinkClassName = functionDetailsBuilder.getSink().getClassName();
- String argClassName = FunctionCommon.getTypeArg(sinkClassName, Sink.class, classLoader).getName();
- functionDetailsBuilder.setSink(functionDetailsBuilder.getSinkBuilder().setTypeClassName(argClassName));
-
- // if source-class not present then set same arg as sink
- if (!functionDetailsBuilder.hasSource() || isBlank(functionDetailsBuilder.getSource().getClassName())) {
- functionDetailsBuilder
- .setSource(functionDetailsBuilder.getSourceBuilder().setTypeClassName(argClassName));
- }
-
- } catch (IllegalArgumentException ie) {
- throw ie;
- } catch (Exception e) {
- log.error("Failed to validate sink class", e);
- throw new IllegalArgumentException("Failed to validate sink class-name", e);
+ "Schema type mismatch " + typeArg.getActualName() + " vs " + schemaTypeArg.getActualName());
}
- } else if (isBlank(functionDetailsBuilder.getSinkBuilder().getTypeClassName())) {
- // if function-sink-class is not present then set function-sink type-class according to function class
- functionDetailsBuilder
- .setSink(functionDetailsBuilder.getSinkBuilder().setTypeClassName(typeArgs[1].getName()));
}
}
}
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java
index 8d621cc965f..54f3da7a448 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java
@@ -19,14 +19,50 @@
package org.apache.pulsar.functions.utils.functions;
import java.nio.file.Path;
-import lombok.Builder;
-import lombok.Data;
import org.apache.pulsar.common.functions.FunctionDefinition;
+import org.apache.pulsar.functions.utils.FunctionFilePackage;
+import org.apache.pulsar.functions.utils.ValidatableFunctionPackage;
-@Builder
-@Data
-public class FunctionArchive {
- private Path archivePath;
- private ClassLoader classLoader;
- private FunctionDefinition functionDefinition;
+public class FunctionArchive implements AutoCloseable {
+ private final Path archivePath;
+ private final FunctionDefinition functionDefinition;
+ private final String narExtractionDirectory;
+ private final boolean enableClassloading;
+ private ValidatableFunctionPackage functionPackage;
+ private boolean closed;
+
+ public FunctionArchive(Path archivePath, FunctionDefinition functionDefinition, String narExtractionDirectory,
+ boolean enableClassloading) {
+ this.archivePath = archivePath;
+ this.functionDefinition = functionDefinition;
+ this.narExtractionDirectory = narExtractionDirectory;
+ this.enableClassloading = enableClassloading;
+ }
+
+ public Path getArchivePath() {
+ return archivePath;
+ }
+
+ public synchronized ValidatableFunctionPackage getFunctionPackage() {
+ if (closed) {
+ throw new IllegalStateException("FunctionArchive is already closed");
+ }
+ if (functionPackage == null) {
+ functionPackage = new FunctionFilePackage(archivePath.toFile(), narExtractionDirectory, enableClassloading,
+ FunctionDefinition.class);
+ }
+ return functionPackage;
+ }
+
+ public FunctionDefinition getFunctionDefinition() {
+ return functionDefinition;
+ }
+
+ @Override
+ public synchronized void close() throws Exception {
+ closed = true;
+ if (functionPackage instanceof AutoCloseable) {
+ ((AutoCloseable) functionPackage).close();
+ }
+ }
}
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java
index 8b70a4b6508..e09f3f1f12f 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java
@@ -16,6 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.pulsar.functions.utils.functions;
import java.io.File;
@@ -30,10 +31,8 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.pulsar.common.functions.FunctionDefinition;
import org.apache.pulsar.common.nar.NarClassLoader;
-import org.apache.pulsar.common.nar.NarClassLoaderBuilder;
import org.apache.pulsar.common.util.ObjectMapperFactory;
-import org.apache.pulsar.functions.api.Function;
-import org.apache.pulsar.functions.utils.Exceptions;
+import org.zeroturnaround.zip.ZipUtil;
@UtilityClass
@@ -45,43 +44,40 @@ public class FunctionUtils {
/**
* Extract the Pulsar Function class from a function or archive.
*/
- public static String getFunctionClass(ClassLoader classLoader) throws IOException {
- NarClassLoader ncl = (NarClassLoader) classLoader;
- String configStr = ncl.getServiceDefinition(PULSAR_IO_SERVICE_NAME);
-
- FunctionDefinition conf = ObjectMapperFactory.getThreadLocalYaml().readValue(configStr,
- FunctionDefinition.class);
- if (StringUtils.isEmpty(conf.getFunctionClass())) {
- throw new IOException(
- String.format("The '%s' functionctor does not provide a function implementation", conf.getName()));
- }
+ public static String getFunctionClass(File narFile) throws IOException {
+ return getFunctionDefinition(narFile).getFunctionClass();
+ }
- try {
- // Try to load source class and check it implements Function interface
- Class functionClass = ncl.loadClass(conf.getFunctionClass());
- if (!(Function.class.isAssignableFrom(functionClass))) {
- throw new IOException(
- "Class " + conf.getFunctionClass() + " does not implement interface " + Function.class
- .getName());
- }
- } catch (Throwable t) {
- Exceptions.rethrowIOException(t);
+ public static FunctionDefinition getFunctionDefinition(File narFile) throws IOException {
+ return getPulsarIOServiceConfig(narFile, FunctionDefinition.class);
+ }
+
+ public static <T> T getPulsarIOServiceConfig(File narFile, Class<T> valueType) throws IOException {
+ String filename = "META-INF/services/" + PULSAR_IO_SERVICE_NAME;
+ byte[] configEntry = ZipUtil.unpackEntry(narFile, filename);
+ if (configEntry != null) {
+ return ObjectMapperFactory.getThreadLocalYaml().reader().readValue(configEntry, valueType);
+ } else {
+ return null;
}
+ }
- return conf.getFunctionClass();
+ public static String getFunctionClass(NarClassLoader narClassLoader) throws IOException {
+ return getFunctionDefinition(narClassLoader).getFunctionClass();
}
public static FunctionDefinition getFunctionDefinition(NarClassLoader narClassLoader) throws IOException {
- String configStr = narClassLoader.getServiceDefinition(PULSAR_IO_SERVICE_NAME);
- return ObjectMapperFactory.getThreadLocalYaml().readValue(configStr, FunctionDefinition.class);
+ return getPulsarIOServiceConfig(narClassLoader, FunctionDefinition.class);
}
- public static TreeMap<String, FunctionArchive> searchForFunctions(String functionsDirectory) throws IOException {
- return searchForFunctions(functionsDirectory, false);
+ public static <T> T getPulsarIOServiceConfig(NarClassLoader narClassLoader, Class<T> valueType) throws IOException {
+ return ObjectMapperFactory.getThreadLocalYaml().reader()
+ .readValue(narClassLoader.getServiceDefinition(PULSAR_IO_SERVICE_NAME), valueType);
}
public static TreeMap<String, FunctionArchive> searchForFunctions(String functionsDirectory,
- boolean alwaysPopulatePath) throws IOException {
+ String narExtractionDirectory,
+ boolean enableClassloading) throws IOException {
Path path = Paths.get(functionsDirectory).toAbsolutePath();
log.info("Searching for functions in {}", path);
@@ -95,22 +91,12 @@ public class FunctionUtils {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(path, "*.nar")) {
for (Path archive : stream) {
try {
-
- NarClassLoader ncl = NarClassLoaderBuilder.builder()
- .narFile(new File(archive.toString()))
- .build();
-
- FunctionArchive.FunctionArchiveBuilder functionArchiveBuilder = FunctionArchive.builder();
- FunctionDefinition cntDef = FunctionUtils.getFunctionDefinition(ncl);
+ FunctionDefinition cntDef = FunctionUtils.getFunctionDefinition(archive.toFile());
log.info("Found function {} from {}", cntDef, archive);
-
- functionArchiveBuilder.archivePath(archive);
-
- functionArchiveBuilder.classLoader(ncl);
- functionArchiveBuilder.functionDefinition(cntDef);
-
- if (alwaysPopulatePath || !StringUtils.isEmpty(cntDef.getFunctionClass())) {
- functions.put(cntDef.getName(), functionArchiveBuilder.build());
+ if (!StringUtils.isEmpty(cntDef.getFunctionClass())) {
+ FunctionArchive functionArchive =
+ new FunctionArchive(archive, cntDef, narExtractionDirectory, enableClassloading);
+ functions.put(cntDef.getName(), functionArchive);
}
} catch (Throwable t) {
log.warn("Failed to load function from {}", archive, t);
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java
index e2e6e39e273..b78c5c45fbd 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java
@@ -20,17 +20,79 @@ package org.apache.pulsar.functions.utils.io;
import java.nio.file.Path;
import java.util.List;
-import lombok.Builder;
-import lombok.Data;
+import org.apache.commons.lang3.StringUtils;
import org.apache.pulsar.common.io.ConfigFieldDefinition;
import org.apache.pulsar.common.io.ConnectorDefinition;
+import org.apache.pulsar.functions.utils.FunctionFilePackage;
+import org.apache.pulsar.functions.utils.ValidatableFunctionPackage;
-@Builder
-@Data
-public class Connector {
- private Path archivePath;
+public class Connector implements AutoCloseable {
+ private final Path archivePath;
+ private final String narExtractionDirectory;
+ private final boolean enableClassloading;
+ private ValidatableFunctionPackage connectorFunctionPackage;
private List<ConfigFieldDefinition> sourceConfigFieldDefinitions;
private List<ConfigFieldDefinition> sinkConfigFieldDefinitions;
- private ClassLoader classLoader;
private ConnectorDefinition connectorDefinition;
+ private boolean closed;
+
+ public Connector(Path archivePath, ConnectorDefinition connectorDefinition, String narExtractionDirectory,
+ boolean enableClassloading) {
+ this.archivePath = archivePath;
+ this.connectorDefinition = connectorDefinition;
+ this.narExtractionDirectory = narExtractionDirectory;
+ this.enableClassloading = enableClassloading;
+ }
+
+ public Path getArchivePath() {
+ return archivePath;
+ }
+
+ public synchronized ValidatableFunctionPackage getConnectorFunctionPackage() {
+ checkState();
+ if (connectorFunctionPackage == null) {
+ connectorFunctionPackage =
+ new FunctionFilePackage(archivePath.toFile(), narExtractionDirectory, enableClassloading,
+ ConnectorDefinition.class);
+ }
+ return connectorFunctionPackage;
+ }
+
+ private void checkState() {
+ if (closed) {
+ throw new IllegalStateException("Connector is already closed");
+ }
+ }
+
+ public synchronized List<ConfigFieldDefinition> getSourceConfigFieldDefinitions() {
+ checkState();
+ if (sourceConfigFieldDefinitions == null && !StringUtils.isEmpty(connectorDefinition.getSourceClass())
+ && !StringUtils.isEmpty(connectorDefinition.getSourceConfigClass())) {
+ sourceConfigFieldDefinitions = ConnectorUtils.getConnectorConfigDefinition(getConnectorFunctionPackage(),
+ connectorDefinition.getSourceConfigClass());
+ }
+ return sourceConfigFieldDefinitions;
+ }
+
+ public synchronized List<ConfigFieldDefinition> getSinkConfigFieldDefinitions() {
+ checkState();
+ if (sinkConfigFieldDefinitions == null && !StringUtils.isEmpty(connectorDefinition.getSinkClass())
+ && !StringUtils.isEmpty(connectorDefinition.getSinkConfigClass())) {
+ sinkConfigFieldDefinitions = ConnectorUtils.getConnectorConfigDefinition(getConnectorFunctionPackage(),
+ connectorDefinition.getSinkConfigClass());
+ }
+ return sinkConfigFieldDefinitions;
+ }
+
+ public ConnectorDefinition getConnectorDefinition() {
+ return connectorDefinition;
+ }
+
+ @Override
+ public synchronized void close() throws Exception {
+ closed = true;
+ if (connectorFunctionPackage instanceof AutoCloseable) {
+ ((AutoCloseable) connectorFunctionPackage).close();
+ }
+ }
}
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java
index a4c74d0cdfd..24d4f8fbf68 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java
@@ -20,9 +20,6 @@ package org.apache.pulsar.functions.utils.io;
import java.io.File;
import java.io.IOException;
-import java.lang.annotation.Annotation;
-import java.lang.reflect.Field;
-import java.lang.reflect.Method;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -34,14 +31,18 @@ import java.util.Map;
import java.util.TreeMap;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
+import net.bytebuddy.description.annotation.AnnotationDescription;
+import net.bytebuddy.description.annotation.AnnotationValue;
+import net.bytebuddy.description.field.FieldDescription;
+import net.bytebuddy.description.method.MethodDescription;
+import net.bytebuddy.description.type.TypeDefinition;
import org.apache.commons.lang3.StringUtils;
import org.apache.pulsar.common.io.ConfigFieldDefinition;
import org.apache.pulsar.common.io.ConnectorDefinition;
import org.apache.pulsar.common.nar.NarClassLoader;
-import org.apache.pulsar.common.nar.NarClassLoaderBuilder;
-import org.apache.pulsar.common.util.ObjectMapperFactory;
-import org.apache.pulsar.common.util.Reflections;
import org.apache.pulsar.functions.utils.Exceptions;
+import org.apache.pulsar.functions.utils.ValidatableFunctionPackage;
+import org.apache.pulsar.functions.utils.functions.FunctionUtils;
import org.apache.pulsar.io.core.BatchSource;
import org.apache.pulsar.io.core.Sink;
import org.apache.pulsar.io.core.Source;
@@ -68,7 +69,7 @@ public class ConnectorUtils {
Class sourceClass = narClassLoader.loadClass(conf.getSourceClass());
if (!(Source.class.isAssignableFrom(sourceClass) || BatchSource.class.isAssignableFrom(sourceClass))) {
throw new IOException(String.format("Class %s does not implement interface %s or %s",
- conf.getSourceClass(), Source.class.getName(), BatchSource.class.getName()));
+ conf.getSourceClass(), Source.class.getName(), BatchSource.class.getName()));
}
} catch (Throwable t) {
Exceptions.rethrowIOException(t);
@@ -101,32 +102,36 @@ public class ConnectorUtils {
return conf.getSinkClass();
}
- public static ConnectorDefinition getConnectorDefinition(NarClassLoader narClassLoader) throws IOException {
- String configStr = narClassLoader.getServiceDefinition(PULSAR_IO_SERVICE_NAME);
+ public static ConnectorDefinition getConnectorDefinition(File narFile) throws IOException {
+ return FunctionUtils.getPulsarIOServiceConfig(narFile, ConnectorDefinition.class);
+ }
- return ObjectMapperFactory.getThreadLocalYaml().readValue(configStr, ConnectorDefinition.class);
+ public static ConnectorDefinition getConnectorDefinition(NarClassLoader narClassLoader) throws IOException {
+ return FunctionUtils.getPulsarIOServiceConfig(narClassLoader, ConnectorDefinition.class);
}
- public static List<ConfigFieldDefinition> getConnectorConfigDefinition(ClassLoader classLoader,
- String configClassName) throws Exception {
+ public static List<ConfigFieldDefinition> getConnectorConfigDefinition(
+ ValidatableFunctionPackage connectorFunctionPackage,
+ String configClassName) {
List<ConfigFieldDefinition> retval = new LinkedList<>();
- Class configClass = classLoader.loadClass(configClassName);
- for (Field field : Reflections.getAllFields(configClass)) {
- if (java.lang.reflect.Modifier.isStatic(field.getModifiers())) {
- // We dont want static fields
+ TypeDefinition configClass = connectorFunctionPackage.resolveType(configClassName);
+
+ for (FieldDescription field : getAllFields(configClass)) {
+ if (field.isStatic()) {
+ // We don't want static fields
continue;
}
- field.setAccessible(true);
ConfigFieldDefinition configFieldDefinition = new ConfigFieldDefinition();
configFieldDefinition.setFieldName(field.getName());
- configFieldDefinition.setTypeName(field.getType().getName());
+ configFieldDefinition.setTypeName(field.getType().getActualName());
Map<String, String> attributes = new HashMap<>();
- for (Annotation annotation : field.getAnnotations()) {
- if (annotation.annotationType().equals(FieldDoc.class)) {
- FieldDoc fieldDoc = (FieldDoc) annotation;
- for (Method method : FieldDoc.class.getDeclaredMethods()) {
- Object value = method.invoke(fieldDoc);
- attributes.put(method.getName(), value == null ? "" : value.toString());
+ for (AnnotationDescription annotation : field.getDeclaredAnnotations()) {
+ if (annotation.getAnnotationType().represents(FieldDoc.class)) {
+ for (MethodDescription.InDefinedShape method : annotation.getAnnotationType()
+ .getDeclaredMethods()) {
+ AnnotationValue<?, ?> value = annotation.getValue(method.getName());
+ attributes.put(method.getName(),
+ value == null || value.resolve() == null ? "" : value.resolve().toString());
}
}
}
@@ -137,12 +142,25 @@ public class ConnectorUtils {
return retval;
}
+ private static List<FieldDescription> getAllFields(TypeDefinition type) {
+ List<FieldDescription> fields = new LinkedList<>();
+ fields.addAll(type.getDeclaredFields());
+
+ if (type.getSuperClass() != null) {
+ fields.addAll(getAllFields(type.getSuperClass()));
+ }
+
+ return fields;
+ }
+
public static TreeMap<String, Connector> searchForConnectors(String connectorsDirectory,
- String narExtractionDirectory) throws IOException {
+ String narExtractionDirectory,
+ boolean enableClassloading) throws IOException {
Path path = Paths.get(connectorsDirectory).toAbsolutePath();
log.info("Searching for connectors in {}", path);
TreeMap<String, Connector> connectors = new TreeMap<>();
+
if (!path.toFile().exists()) {
log.warn("Connectors archive directory not found");
return connectors;
@@ -151,40 +169,15 @@ public class ConnectorUtils {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(path, "*.nar")) {
for (Path archive : stream) {
try {
-
- NarClassLoader ncl = NarClassLoaderBuilder.builder()
- .narFile(new File(archive.toString()))
- .extractionDirectory(narExtractionDirectory)
- .build();
-
- Connector.ConnectorBuilder connectorBuilder = Connector.builder();
- ConnectorDefinition cntDef = ConnectorUtils.getConnectorDefinition(ncl);
+ ConnectorDefinition cntDef = ConnectorUtils.getConnectorDefinition(archive.toFile());
log.info("Found connector {} from {}", cntDef, archive);
-
- connectorBuilder.archivePath(archive);
- if (!StringUtils.isEmpty(cntDef.getSourceClass())) {
- if (!StringUtils.isEmpty(cntDef.getSourceConfigClass())) {
- connectorBuilder.sourceConfigFieldDefinitions(ConnectorUtils
- .getConnectorConfigDefinition(ncl, cntDef.getSourceConfigClass()));
- }
- }
-
- if (!StringUtils.isEmpty(cntDef.getSinkClass())) {
- if (!StringUtils.isEmpty(cntDef.getSinkConfigClass())) {
- connectorBuilder.sinkConfigFieldDefinitions(
- ConnectorUtils.getConnectorConfigDefinition(ncl, cntDef.getSinkConfigClass()));
- }
- }
-
- connectorBuilder.classLoader(ncl);
- connectorBuilder.connectorDefinition(cntDef);
- connectors.put(cntDef.getName(), connectorBuilder.build());
+ Connector connector = new Connector(archive, cntDef, narExtractionDirectory, enableClassloading);
+ connectors.put(cntDef.getName(), connector);
} catch (Throwable t) {
log.warn("Failed to load connector from {}", archive, t);
}
}
-
- return connectors;
}
+ return connectors;
}
}
diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java
index b991f8b4f47..98c8dacb25c 100644
--- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java
+++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java
@@ -19,9 +19,16 @@
package org.apache.pulsar.functions.utils;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import java.io.File;
import java.util.Collection;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import net.bytebuddy.description.type.TypeDefinition;
+import net.bytebuddy.pool.TypePool;
import org.apache.pulsar.client.impl.MessageIdImpl;
-import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.functions.api.Context;
import org.apache.pulsar.functions.api.Function;
import org.apache.pulsar.functions.api.Record;
@@ -31,52 +38,10 @@ import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import java.io.File;
-import java.util.UUID;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertEquals;
-
/**
* Unit test of {@link Exceptions}.
*/
public class FunctionCommonTest {
-
- @Test
- public void testValidateLocalFileUrl() throws Exception {
- String fileLocation = FutureUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath();
- try {
- // eg: fileLocation : /dir/fileName.jar (invalid)
- FunctionCommon.extractClassLoader(fileLocation);
- Assert.fail("should fail with invalid url: without protocol");
- } catch (IllegalArgumentException ie) {
- // Ok.. expected exception
- }
- String fileLocationWithProtocol = "file://" + fileLocation;
- // eg: fileLocation : file:///dir/fileName.jar (valid)
- FunctionCommon.extractClassLoader(fileLocationWithProtocol);
- // eg: fileLocation : file:/dir/fileName.jar (valid)
- fileLocationWithProtocol = "file:" + fileLocation;
- FunctionCommon.extractClassLoader(fileLocationWithProtocol);
- }
-
- @Test
- public void testValidateHttpFileUrl() throws Exception {
-
- String jarHttpUrl = "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar";
- FunctionCommon.extractClassLoader(jarHttpUrl);
-
- jarHttpUrl = "http://_invalidurl_.com";
- try {
- // eg: fileLocation : /dir/fileName.jar (invalid)
- FunctionCommon.extractClassLoader(jarHttpUrl);
- Assert.fail("should fail with invalid url: without protocol");
- } catch (Exception ie) {
- // Ok.. expected exception
- }
- }
-
@Test
public void testDownloadFile() throws Exception {
String jarHttpUrl = "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar";
@@ -128,6 +93,14 @@ public class FunctionCommonTest {
}
}, false
},
+ {
+ new Function<String, CompletableFuture<Integer>>() {
+ @Override
+ public CompletableFuture<Integer> process(String input, Context context) throws Exception {
+ return null;
+ }
+ }, false
+ },
{
new java.util.function.Function<String, Integer>() {
@Override
@@ -144,6 +117,14 @@ public class FunctionCommonTest {
}
}, false
},
+ {
+ new java.util.function.Function<String, CompletableFuture<Integer>>() {
+ @Override
+ public CompletableFuture<Integer> apply(String s) {
+ return null;
+ }
+ }, false
+ },
{
new WindowFunction<String, Integer>() {
@Override
@@ -160,6 +141,14 @@ public class FunctionCommonTest {
}
}, true
},
+ {
+ new WindowFunction<String, CompletableFuture<Integer>>() {
+ @Override
+ public CompletableFuture<Integer> process(Collection<Record<String>> input, WindowContext context) throws Exception {
+ return null;
+ }
+ }, true
+ },
{
new java.util.function.Function<Collection<String>, Integer>() {
@Override
@@ -175,15 +164,26 @@ public class FunctionCommonTest {
return null;
}
}, true
+ },
+ {
+ new java.util.function.Function<Collection<String>, CompletableFuture<Integer>>() {
+ @Override
+ public CompletableFuture<Integer> apply(Collection<String> strings) {
+ return null;
+ }
+ }, true
}
};
}
@Test(dataProvider = "function")
public void testGetFunctionTypes(Object function, boolean isWindowConfigPresent) {
- Class<?>[] types = FunctionCommon.getFunctionTypes(function.getClass(), isWindowConfigPresent);
+ TypePool typePool = TypePool.Default.of(function.getClass().getClassLoader());
+ TypeDefinition[] types =
+ FunctionCommon.getFunctionTypes(typePool.describe(function.getClass().getName()).resolve(),
+ isWindowConfigPresent);
assertEquals(types.length, 2);
- assertEquals(types[0], String.class);
- assertEquals(types[1], Integer.class);
+ assertEquals(types[0].asErasure().getTypeName(), String.class.getName());
+ assertEquals(types[1].asErasure().getTypeName(), Integer.class.getName());
}
}
diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java
index d2b387cdf99..85d481c2524 100644
--- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java
+++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java
@@ -18,12 +18,23 @@
*/
package org.apache.pulsar.functions.utils;
+import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE;
+import static org.apache.pulsar.common.functions.FunctionConfig.Runtime.PYTHON;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertThrows;
+import static org.testng.Assert.assertTrue;
import com.google.gson.Gson;
-
-import org.apache.pulsar.client.api.SubscriptionInitialPosition;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.client.api.SubscriptionInitialPosition;
import org.apache.pulsar.client.impl.schema.JSONSchema;
import org.apache.pulsar.common.functions.ConsumerConfig;
import org.apache.pulsar.common.functions.FunctionConfig;
@@ -31,28 +42,29 @@ import org.apache.pulsar.common.functions.ProducerConfig;
import org.apache.pulsar.common.functions.Resources;
import org.apache.pulsar.common.functions.WindowConfig;
import org.apache.pulsar.common.util.Reflections;
+import org.apache.pulsar.functions.api.Record;
+import org.apache.pulsar.functions.api.WindowContext;
+import org.apache.pulsar.functions.api.WindowFunction;
import org.apache.pulsar.functions.api.utils.IdentityFunction;
import org.apache.pulsar.functions.proto.Function;
import org.apache.pulsar.functions.proto.Function.FunctionDetails;
import org.testng.annotations.Test;
-import java.lang.reflect.Field;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE;
-import static org.apache.pulsar.common.functions.FunctionConfig.Runtime.PYTHON;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertThrows;
-import static org.testng.Assert.assertTrue;
-
/**
* Unit test of {@link Reflections}.
*/
@Slf4j
public class FunctionConfigUtilsTest {
+ public static class WordCountWindowFunction implements WindowFunction<String, Void> {
+ @Override
+ public Void process(Collection<Record<String>> inputs, WindowContext context) throws Exception {
+ for (Record<String> input : inputs) {
+ Arrays.asList(input.getValue().split("\\.")).forEach(word -> context.incrCounter(word, 1));
+ }
+ return null;
+ }
+ }
+
@Test
public void testAutoAckConvertFailed() {
@@ -62,7 +74,7 @@ public class FunctionConfigUtilsTest {
functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATMOST_ONCE);
assertThrows(IllegalArgumentException.class, () -> {
- FunctionConfigUtils.convert(functionConfig, (ClassLoader) null);
+ FunctionConfigUtils.convert(functionConfig);
});
}
@@ -97,7 +109,7 @@ public class FunctionConfigUtilsTest {
producerConfig.setUseThreadLocalProducers(true);
producerConfig.setBatchBuilder("DEFAULT");
functionConfig.setProducerConfig(producerConfig);
- Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null);
+ Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig);
FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails);
// add default resources
@@ -117,7 +129,7 @@ public class FunctionConfigUtilsTest {
functionConfig.setNamespace("test-namespace");
functionConfig.setName("test-function");
functionConfig.setParallelism(1);
- functionConfig.setClassName(IdentityFunction.class.getName());
+ functionConfig.setClassName(WordCountWindowFunction.class.getName());
Map<String, ConsumerConfig> inputSpecs = new HashMap<>();
inputSpecs.put("test-input", ConsumerConfig.builder().isRegexPattern(true).serdeClassName("test-serde").build());
functionConfig.setInputSpecs(inputSpecs);
@@ -138,7 +150,7 @@ public class FunctionConfigUtilsTest {
producerConfig.setUseThreadLocalProducers(true);
producerConfig.setBatchBuilder("KEY_BASED");
functionConfig.setProducerConfig(producerConfig);
- Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null);
+ Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig);
FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails);
// WindowsFunction guarantees convert to FunctionGuarantees.
@@ -160,7 +172,7 @@ public class FunctionConfigUtilsTest {
FunctionConfig functionConfig = createFunctionConfig();
functionConfig.setBatchBuilder("KEY_BASED");
- Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null);
+ Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig);
assertEquals(functionDetails.getSink().getProducerSpec().getBatchBuilder(), "KEY_BASED");
FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails);
@@ -515,7 +527,6 @@ public class FunctionConfigUtilsTest {
functionConfig.setUserConfig(new HashMap<>());
functionConfig.setAutoAck(true);
functionConfig.setTimeoutMs(2000L);
- functionConfig.setWindowConfig(new WindowConfig().setWindowLengthCount(10));
functionConfig.setCleanupSubscription(true);
functionConfig.setRuntimeFlags("-Dfoo=bar");
return functionConfig;
@@ -549,7 +560,7 @@ public class FunctionConfigUtilsTest {
config.setForwardSourceMessageProperty(true);
FunctionConfigUtils.inferMissingArguments(config, false);
assertNull(config.getForwardSourceMessageProperty());
- FunctionDetails details = FunctionConfigUtils.convert(config, FunctionConfigUtilsTest.class.getClassLoader());
+ FunctionDetails details = FunctionConfigUtils.convert(config);
assertFalse(details.getSink().getForwardSourceMessageProperty());
String detailsJson = "'" + JsonFormat.printer().omittingInsignificantWhitespace().print(details) + "'";
log.info("Function details : {}", detailsJson);
@@ -636,7 +647,7 @@ public class FunctionConfigUtilsTest {
@Test
public void testPoolMessages() {
FunctionConfig functionConfig = createFunctionConfig();
- Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null);
+ Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig);
assertFalse(functionDetails.getSource().getInputSpecsMap().get("test-input").getPoolMessages());
FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails);
assertFalse(convertedConfig.getInputSpecs().get("test-input").isPoolMessages());
@@ -646,7 +657,7 @@ public class FunctionConfigUtilsTest {
.poolMessages(true).build());
functionConfig.setInputSpecs(inputSpecs);
- functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null);
+ functionDetails = FunctionConfigUtils.convert(functionConfig);
assertTrue(functionDetails.getSource().getInputSpecsMap().get("test-input").getPoolMessages());
convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails);
diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java
index 701e852fa7d..cb55eebbe50 100644
--- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java
+++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java
@@ -18,36 +18,38 @@
*/
package org.apache.pulsar.functions.utils;
+import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE;
+import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.ATMOST_ONCE;
+import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertThrows;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.expectThrows;
import com.google.common.collect.Lists;
import com.google.gson.Gson;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.Accessors;
import org.apache.pulsar.common.functions.ConsumerConfig;
import org.apache.pulsar.common.functions.FunctionConfig;
import org.apache.pulsar.common.functions.Resources;
+import org.apache.pulsar.common.io.ConnectorDefinition;
import org.apache.pulsar.common.io.SinkConfig;
import org.apache.pulsar.config.validation.ConfigValidationAnnotations;
-import org.apache.pulsar.functions.api.utils.IdentityFunction;
+import org.apache.pulsar.functions.api.Record;
import org.apache.pulsar.functions.proto.Function;
+import org.apache.pulsar.io.core.Sink;
+import org.apache.pulsar.io.core.SinkContext;
import org.testng.annotations.Test;
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE;
-import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.ATMOST_ONCE;
-import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertThrows;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.expectThrows;
-
/**
* Unit test of {@link SinkConfigUtilsTest}.
*/
@@ -61,6 +63,27 @@ public class SinkConfigUtilsTest {
private String configParameter;
}
+
+ public static class NopSink implements Sink<Object> {
+
+ @Override
+ public void open(Map<String, Object> config, SinkContext sinkContext) throws Exception {
+
+ }
+
+ @Override
+ public void write(Record<Object> record) throws Exception {
+
+ }
+
+ @Override
+ public void close() throws Exception {
+
+ }
+ }
+
+
+
@Test
public void testAutoAckConvertFailed() throws IOException {
@@ -147,7 +170,7 @@ public class SinkConfigUtilsTest {
SinkConfig sinkConfig = createSinkConfig();
sinkConfig.setRetainOrdering(testcase);
Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null));
- assertEquals(functionDetails.getRetainOrdering(), testcase != null ? testcase : false);
+ assertEquals(functionDetails.getRetainOrdering(), testcase != null ? testcase.booleanValue() : false);
SinkConfig result = SinkConfigUtils.convertFromDetails(functionDetails);
assertEquals(result.getRetainOrdering(), testcase != null ? testcase : Boolean.valueOf(false));
}
@@ -160,7 +183,7 @@ public class SinkConfigUtilsTest {
SinkConfig sinkConfig = createSinkConfig();
sinkConfig.setRetainKeyOrdering(testcase);
Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null));
- assertEquals(functionDetails.getRetainKeyOrdering(), testcase != null ? testcase : false);
+ assertEquals(functionDetails.getRetainKeyOrdering(), testcase != null ? testcase.booleanValue() : false);
SinkConfig result = SinkConfigUtils.convertFromDetails(functionDetails);
assertEquals(result.getRetainKeyOrdering(), testcase != null ? testcase : Boolean.valueOf(false));
}
@@ -276,6 +299,24 @@ public class SinkConfigUtilsTest {
assertEquals(sinkConfig.getInputSpecs().get("test-input").getReceiverQueueSize().intValue(), 1000);
}
+ @Test
+ public void testMergeDifferentInputSpecWithInputsSet() {
+ SinkConfig sinkConfig = createSinkConfig();
+ sinkConfig.getInputSpecs().put("test-input", ConsumerConfig.builder().isRegexPattern(false).receiverQueueSize(1000).build());
+
+ Map<String, ConsumerConfig> inputSpecs = new HashMap<>();
+ ConsumerConfig newConsumerConfig = ConsumerConfig.builder().isRegexPattern(false).serdeClassName("test-serde").receiverQueueSize(58).build();
+ inputSpecs.put("test-input", newConsumerConfig);
+ SinkConfig newSinkConfig = createUpdatedSinkConfig("inputSpecs", inputSpecs);
+ newSinkConfig.setInputs(new ArrayList<>());
+ newSinkConfig.getInputs().add("test-input");
+ SinkConfig mergedConfig = SinkConfigUtils.validateUpdate(sinkConfig, newSinkConfig);
+ assertEquals(mergedConfig.getInputSpecs().get("test-input"), newConsumerConfig);
+
+ // make sure original sinkConfig was not modified
+ assertEquals(sinkConfig.getInputSpecs().get("test-input").getReceiverQueueSize().intValue(), 1000);
+ }
+
@Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "Processing Guarantees cannot be altered")
public void testMergeDifferentProcessingGuarantees() {
SinkConfig sinkConfig = createSinkConfig();
@@ -448,7 +489,7 @@ public class SinkConfigUtilsTest {
sinkConfig.setNamespace("test-namespace");
sinkConfig.setName("test-sink");
sinkConfig.setParallelism(1);
- sinkConfig.setClassName(IdentityFunction.class.getName());
+ sinkConfig.setClassName(NopSink.class.getName());
Map<String, ConsumerConfig> inputSpecs = new HashMap<>();
inputSpecs.put("test-input", ConsumerConfig.builder().isRegexPattern(true).serdeClassName("test-serde").build());
sinkConfig.setInputSpecs(inputSpecs);
@@ -501,13 +542,13 @@ public class SinkConfigUtilsTest {
SinkConfig sinkConfig = createSinkConfig();
sinkConfig.setInputSpecs(null);
sinkConfig.setTopicsPattern("my-topic-*");
- SinkConfigUtils.validateAndExtractDetails(sinkConfig, this.getClass().getClassLoader(),
- true);
+ LoadedFunctionPackage validatableFunction =
+ new LoadedFunctionPackage(this.getClass().getClassLoader(), ConnectorDefinition.class);
+
+ SinkConfigUtils.validateAndExtractDetails(sinkConfig, validatableFunction, true);
sinkConfig.setTimeoutMs(null);
- SinkConfigUtils.validateAndExtractDetails(sinkConfig, this.getClass().getClassLoader(),
- true);
+ SinkConfigUtils.validateAndExtractDetails(sinkConfig, validatableFunction, true);
sinkConfig.setTimeoutMs(0L);
- SinkConfigUtils.validateAndExtractDetails(sinkConfig, this.getClass().getClassLoader(),
- true);
+ SinkConfigUtils.validateAndExtractDetails(sinkConfig, validatableFunction, true);
}
}
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
index 01b240d7f97..542eb30062a 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
@@ -69,6 +69,7 @@ import org.apache.pulsar.functions.runtime.RuntimeSpawner;
import org.apache.pulsar.functions.utils.Actions;
import org.apache.pulsar.functions.utils.FunctionCommon;
import org.apache.pulsar.functions.utils.SourceConfigUtils;
+import org.apache.pulsar.functions.utils.ValidatableFunctionPackage;
import org.apache.pulsar.functions.utils.io.Connector;
@Data
@@ -496,7 +497,7 @@ public class FunctionActioner {
builder.setClassName(sourceClass);
functionDetails.setSource(builder);
- fillSourceTypeClass(functionDetails, connector.getClassLoader(), sourceClass);
+ fillSourceTypeClass(functionDetails, connector.getConnectorFunctionPackage(), sourceClass);
return archive;
}
}
@@ -512,7 +513,7 @@ public class FunctionActioner {
builder.setClassName(sinkClass);
functionDetails.setSink(builder);
- fillSinkTypeClass(functionDetails, connector.getClassLoader(), sinkClass);
+ fillSinkTypeClass(functionDetails, connector.getConnectorFunctionPackage(), sinkClass);
return archive;
}
}
@@ -525,8 +526,8 @@ public class FunctionActioner {
}
private void fillSourceTypeClass(FunctionDetails.Builder functionDetails,
- ClassLoader narClassLoader, String className) throws ClassNotFoundException {
- String typeArg = getSourceType(className, narClassLoader).getName();
+ ValidatableFunctionPackage functionPackage, String className) {
+ String typeArg = getSourceType(className, functionPackage.getTypePool()).asErasure().getName();
SourceSpec.Builder sourceBuilder = SourceSpec.newBuilder(functionDetails.getSource());
sourceBuilder.setTypeClassName(typeArg);
@@ -541,8 +542,8 @@ public class FunctionActioner {
}
private void fillSinkTypeClass(FunctionDetails.Builder functionDetails,
- ClassLoader narClassLoader, String className) throws ClassNotFoundException {
- String typeArg = getSinkType(className, narClassLoader).getName();
+ ValidatableFunctionPackage functionPackage, String className) {
+ String typeArg = getSinkType(className, functionPackage.getTypePool()).asErasure().getName();
SinkSpec.Builder sinkBuilder = SinkSpec.newBuilder(functionDetails.getSink());
sinkBuilder.setTypeClassName(typeArg);
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java
index 307e63262af..70237305a33 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java
@@ -663,6 +663,14 @@ public class PulsarWorkerService implements WorkerService {
if (statsUpdater != null) {
statsUpdater.shutdownNow();
}
+
+ if (null != functionsManager) {
+ functionsManager.close();
+ }
+
+ if (null != connectorsManager) {
+ connectorsManager.close();
+ }
}
}
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java
index 030367e8864..7f7b3b78efb 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java
@@ -29,7 +29,6 @@ import static org.apache.pulsar.functions.utils.FunctionCommon.getStateNamespace
import static org.apache.pulsar.functions.utils.FunctionCommon.getUniquePackageName;
import static org.apache.pulsar.functions.utils.FunctionCommon.isFunctionCodeBuiltin;
import static org.apache.pulsar.functions.worker.rest.RestUtils.throwUnavailableException;
-import static org.apache.pulsar.functions.worker.rest.api.FunctionsImpl.downloadPackageFile;
import com.google.common.base.Utf8;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufUtil;
@@ -43,6 +42,7 @@ import java.net.URI;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Base64;
import java.util.Collection;
import java.util.LinkedList;
@@ -66,6 +66,7 @@ import org.apache.bookkeeper.clients.config.StorageClientSettings;
import org.apache.bookkeeper.clients.exceptions.NamespaceNotFoundException;
import org.apache.bookkeeper.clients.exceptions.StreamNotFoundException;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
import org.apache.pulsar.broker.authentication.AuthenticationParameters;
import org.apache.pulsar.client.admin.PulsarAdminException;
@@ -100,6 +101,7 @@ import org.apache.pulsar.functions.utils.ComponentTypeUtils;
import org.apache.pulsar.functions.utils.FunctionCommon;
import org.apache.pulsar.functions.utils.FunctionConfigUtils;
import org.apache.pulsar.functions.utils.FunctionMetaDataUtils;
+import org.apache.pulsar.functions.utils.ValidatableFunctionPackage;
import org.apache.pulsar.functions.utils.functions.FunctionArchive;
import org.apache.pulsar.functions.utils.io.Connector;
import org.apache.pulsar.functions.worker.FunctionMetaDataManager;
@@ -1764,9 +1766,88 @@ public abstract class ComponentImpl implements Component<PulsarWorkerService> {
}
}
- protected ClassLoader getClassLoaderFromPackage(String className,
- File packageFile,
- String narExtractionDirectory) {
- return FunctionCommon.getClassLoaderFromPackage(componentType, className, packageFile, narExtractionDirectory);
+ static File downloadPackageFile(PulsarWorkerService worker, String packageName)
+ throws IOException, PulsarAdminException {
+ Path tempDirectory;
+ if (worker.getWorkerConfig().getDownloadDirectory() != null) {
+ tempDirectory = Paths.get(worker.getWorkerConfig().getDownloadDirectory());
+ } else {
+ // use the Nar extraction directory as a temporary directory for downloaded files
+ tempDirectory = Paths.get(worker.getWorkerConfig().getNarExtractionDirectory());
+ }
+ Files.createDirectories(tempDirectory);
+ File file = Files.createTempFile(tempDirectory, "function", ".tmp").toFile();
+ worker.getBrokerAdmin().packages().download(packageName, file.toString());
+ return file;
+ }
+
+ protected File getPackageFile(String functionPkgUrl, String existingPackagePath, InputStream uploadedInputStream)
+ throws IOException, PulsarAdminException {
+ File componentPackageFile = null;
+ if (isNotBlank(functionPkgUrl)) {
+ componentPackageFile = getPackageFile(functionPkgUrl);
+ } else if (existingPackagePath.startsWith(Utils.FILE) || existingPackagePath.startsWith(Utils.HTTP)) {
+ try {
+ componentPackageFile = FunctionCommon.extractFileFromPkgURL(existingPackagePath);
+ } catch (Exception e) {
+ throw new IllegalArgumentException(String.format("Encountered error \"%s\" "
+ + "when getting %s package from %s", e.getMessage(),
+ ComponentTypeUtils.toString(componentType), functionPkgUrl));
+ }
+ } else if (Utils.hasPackageTypePrefix(existingPackagePath)) {
+ componentPackageFile = getPackageFile(existingPackagePath);
+ } else if (uploadedInputStream != null) {
+ componentPackageFile = WorkerUtils.dumpToTmpFile(uploadedInputStream);
+ } else if (!existingPackagePath.startsWith(Utils.BUILTIN)) {
+ componentPackageFile = FunctionCommon.createPkgTempFile();
+ componentPackageFile.deleteOnExit();
+ if (worker().getWorkerConfig().isFunctionsWorkerEnablePackageManagement()) {
+ worker().getBrokerAdmin().packages().download(
+ existingPackagePath,
+ componentPackageFile.getAbsolutePath());
+ } else {
+ WorkerUtils.downloadFromBookkeeper(worker().getDlogNamespace(),
+ componentPackageFile, existingPackagePath);
+ }
+ }
+ return componentPackageFile;
+ }
+
+ protected File downloadPackageFile(String packageName) throws IOException, PulsarAdminException {
+ return downloadPackageFile(worker(), packageName);
+ }
+
+ protected File getPackageFile(String functionPkgUrl) throws IOException, PulsarAdminException {
+ if (Utils.hasPackageTypePrefix(functionPkgUrl)) {
+ return downloadPackageFile(functionPkgUrl);
+ } else {
+ if (!Utils.isFunctionPackageUrlSupported(functionPkgUrl)) {
+ throw new IllegalArgumentException("Function Package url is not valid."
+ + "supported url (http/https/file)");
+ }
+ try {
+ return FunctionCommon.extractFileFromPkgURL(functionPkgUrl);
+ } catch (Exception e) {
+ throw new IllegalArgumentException(String.format("Encountered error \"%s\" "
+ + "when getting %s package from %s", e.getMessage(),
+ ComponentTypeUtils.toString(componentType), functionPkgUrl), e);
+ }
+ }
+ }
+
+ protected ValidatableFunctionPackage getBuiltinFunctionPackage(String archive) {
+ if (!StringUtils.isEmpty(archive)) {
+ if (archive.startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN)) {
+ archive = archive.replaceFirst("^builtin://", "");
+
+ FunctionArchive function = worker().getFunctionsManager().getFunction(archive);
+ // check if builtin connector exists
+ if (function == null) {
+ throw new IllegalArgumentException("Built-in " + componentType + " is not available");
+ }
+ return function.getFunctionPackage();
+ }
+ }
+ return null;
}
}
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
index 4133540cfd1..4f1b4477f1d 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
@@ -28,9 +28,6 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
@@ -44,12 +41,12 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.pulsar.broker.authentication.AuthenticationParameters;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.common.functions.FunctionConfig;
+import org.apache.pulsar.common.functions.FunctionDefinition;
import org.apache.pulsar.common.functions.UpdateOptionsImpl;
import org.apache.pulsar.common.functions.Utils;
import org.apache.pulsar.common.functions.WorkerInfo;
import org.apache.pulsar.common.policies.data.ExceptionInformation;
import org.apache.pulsar.common.policies.data.FunctionStatus;
-import org.apache.pulsar.common.util.ClassLoaderUtils;
import org.apache.pulsar.common.util.RestException;
import org.apache.pulsar.functions.auth.FunctionAuthData;
import org.apache.pulsar.functions.instance.InstanceUtils;
@@ -58,11 +55,14 @@ import org.apache.pulsar.functions.proto.InstanceCommunication;
import org.apache.pulsar.functions.utils.ComponentTypeUtils;
import org.apache.pulsar.functions.utils.FunctionCommon;
import org.apache.pulsar.functions.utils.FunctionConfigUtils;
+import org.apache.pulsar.functions.utils.FunctionFilePackage;
import org.apache.pulsar.functions.utils.FunctionMetaDataUtils;
+import org.apache.pulsar.functions.utils.ValidatableFunctionPackage;
import org.apache.pulsar.functions.utils.functions.FunctionArchive;
import org.apache.pulsar.functions.worker.FunctionMetaDataManager;
import org.apache.pulsar.functions.worker.FunctionsManager;
import org.apache.pulsar.functions.worker.PulsarWorkerService;
+import org.apache.pulsar.functions.worker.WorkerConfig;
import org.apache.pulsar.functions.worker.WorkerUtils;
import org.apache.pulsar.functions.worker.service.api.Functions;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
@@ -767,11 +767,12 @@ public class FunctionsImpl extends ComponentImpl implements Functions<PulsarWork
functionConfig.setTenant(tenant);
functionConfig.setNamespace(namespace);
functionConfig.setName(componentName);
+ WorkerConfig workerConfig = worker().getWorkerConfig();
FunctionConfigUtils.inferMissingArguments(
- functionConfig, worker().getWorkerConfig().isForwardSourceMessageProperty());
+ functionConfig, workerConfig.isForwardSourceMessageProperty());
String archive = functionConfig.getJar();
- ClassLoader classLoader = null;
+ ValidatableFunctionPackage functionPackage = null;
// check if function is builtin and extract classloader
if (!StringUtils.isEmpty(archive)) {
if (archive.startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN)) {
@@ -784,55 +785,39 @@ public class FunctionsImpl extends ComponentImpl implements Functions<PulsarWork
if (function == null) {
throw new IllegalArgumentException(String.format("No Function %s found", archive));
}
- classLoader = function.getClassLoader();
+ functionPackage = function.getFunctionPackage();
}
}
- boolean shouldCloseClassLoader = false;
+ boolean shouldCloseFunctionPackage = false;
try {
-
if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
// if function is not builtin, attempt to extract classloader from package file if it exists
- if (classLoader == null && componentPackageFile != null) {
- classLoader = getClassLoaderFromPackage(functionConfig.getClassName(),
- componentPackageFile, worker().getWorkerConfig().getNarExtractionDirectory());
- shouldCloseClassLoader = true;
+ if (functionPackage == null && componentPackageFile != null) {
+ functionPackage =
+ new FunctionFilePackage(componentPackageFile, workerConfig.getNarExtractionDirectory(),
+ workerConfig.getEnableClassloadingOfExternalFiles(), FunctionDefinition.class);
+ shouldCloseFunctionPackage = true;
}
- if (classLoader == null) {
+ if (functionPackage == null) {
throw new IllegalArgumentException("Function package is not provided");
}
FunctionConfigUtils.ExtractedFunctionDetails functionDetails = FunctionConfigUtils.validateJavaFunction(
- functionConfig, classLoader);
+ functionConfig, functionPackage);
return FunctionConfigUtils.convert(functionConfig, functionDetails);
} else {
- classLoader = FunctionConfigUtils.validate(functionConfig, componentPackageFile);
- shouldCloseClassLoader = true;
- return FunctionConfigUtils.convert(functionConfig, classLoader);
+ FunctionConfigUtils.validateNonJavaFunction(functionConfig);
+ return FunctionConfigUtils.convert(functionConfig);
}
} finally {
- if (shouldCloseClassLoader) {
- ClassLoaderUtils.closeClassLoader(classLoader);
+ if (shouldCloseFunctionPackage && functionPackage instanceof AutoCloseable) {
+ try {
+ ((AutoCloseable) functionPackage).close();
+ } catch (Exception e) {
+ log.error("Failed to close function file", e);
+ }
}
}
}
-
- private File downloadPackageFile(String packageName) throws IOException, PulsarAdminException {
- return downloadPackageFile(worker(), packageName);
- }
-
- static File downloadPackageFile(PulsarWorkerService worker, String packageName)
- throws IOException, PulsarAdminException {
- Path tempDirectory;
- if (worker.getWorkerConfig().getDownloadDirectory() != null) {
- tempDirectory = Paths.get(worker.getWorkerConfig().getDownloadDirectory());
- } else {
- // use the Nar extraction directory as a temporary directory for downloaded files
- tempDirectory = Paths.get(worker.getWorkerConfig().getNarExtractionDirectory());
- }
- Files.createDirectories(tempDirectory);
- File file = Files.createTempFile(tempDirectory, "function", ".tmp").toFile();
- worker.getBrokerAdmin().packages().download(packageName, file.toString());
- return file;
- }
}
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java
index 1b7ba1522bb..8cd020b6b33 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java
@@ -47,19 +47,20 @@ import org.apache.pulsar.common.io.ConnectorDefinition;
import org.apache.pulsar.common.io.SinkConfig;
import org.apache.pulsar.common.policies.data.ExceptionInformation;
import org.apache.pulsar.common.policies.data.SinkStatus;
-import org.apache.pulsar.common.util.ClassLoaderUtils;
import org.apache.pulsar.common.util.RestException;
import org.apache.pulsar.functions.auth.FunctionAuthData;
import org.apache.pulsar.functions.instance.InstanceUtils;
import org.apache.pulsar.functions.proto.Function;
import org.apache.pulsar.functions.proto.InstanceCommunication;
import org.apache.pulsar.functions.utils.ComponentTypeUtils;
-import org.apache.pulsar.functions.utils.FunctionCommon;
+import org.apache.pulsar.functions.utils.FunctionFilePackage;
import org.apache.pulsar.functions.utils.FunctionMetaDataUtils;
import org.apache.pulsar.functions.utils.SinkConfigUtils;
+import org.apache.pulsar.functions.utils.ValidatableFunctionPackage;
import org.apache.pulsar.functions.utils.io.Connector;
import org.apache.pulsar.functions.worker.FunctionMetaDataManager;
import org.apache.pulsar.functions.worker.PulsarWorkerService;
+import org.apache.pulsar.functions.worker.WorkerConfig;
import org.apache.pulsar.functions.worker.WorkerUtils;
import org.apache.pulsar.functions.worker.service.api.Sinks;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
@@ -135,29 +136,14 @@ public class SinksImpl extends ComponentImpl implements Sinks<PulsarWorkerServic
String.format("%s %s already exists", ComponentTypeUtils.toString(componentType), sinkName));
}
- Function.FunctionDetails functionDetails = null;
- boolean isPkgUrlProvided = isNotBlank(sinkPkgUrl);
+ Function.FunctionDetails functionDetails;
File componentPackageFile = null;
try {
// validate parameters
try {
- if (isPkgUrlProvided) {
- if (Utils.hasPackageTypePrefix(sinkPkgUrl)) {
- componentPackageFile = downloadPackageFile(sinkPkgUrl);
- } else {
- if (!Utils.isFunctionPackageUrlSupported(sinkPkgUrl)) {
- throw new IllegalArgumentException(
- "Function Package url is not valid. supported url (http/https/file)");
- }
- try {
- componentPackageFile = FunctionCommon.extractFileFromPkgURL(sinkPkgUrl);
- } catch (Exception e) {
- throw new IllegalArgumentException(
- String.format("Encountered error \"%s\" when getting %s package from %s",
- e.getMessage(), ComponentTypeUtils.toString(componentType), sinkPkgUrl));
- }
- }
+ if (isNotBlank(sinkPkgUrl)) {
+ componentPackageFile = getPackageFile(sinkPkgUrl);
functionDetails = validateUpdateRequestParams(tenant, namespace, sinkName,
sinkConfig, componentPackageFile);
} else {
@@ -235,6 +221,7 @@ public class SinksImpl extends ComponentImpl implements Sinks<PulsarWorkerServic
}
functionMetaDataBuilder.setPackageLocation(packageLocationMetaDataBuilder);
+
updateRequest(null, functionMetaDataBuilder.build());
} finally {
if (componentPackageFile != null && componentPackageFile.exists()) {
@@ -305,72 +292,30 @@ public class SinksImpl extends ComponentImpl implements Sinks<PulsarWorkerServic
throw new RestException(Response.Status.BAD_REQUEST, e.getMessage());
}
- if (existingSinkConfig.equals(mergedConfig) && isBlank(sinkPkgUrl) && uploadedInputStream == null) {
+ if (existingSinkConfig.equals(mergedConfig) && isBlank(sinkPkgUrl) && uploadedInputStream == null
+ && (updateOptions == null || !updateOptions.isUpdateAuthData())) {
log.error("{}/{}/{} Update contains no changes", tenant, namespace, sinkName);
throw new RestException(Response.Status.BAD_REQUEST, "Update contains no change");
}
- Function.FunctionDetails functionDetails = null;
+ Function.FunctionDetails functionDetails;
File componentPackageFile = null;
try {
// validate parameters
try {
- if (isNotBlank(sinkPkgUrl)) {
- if (Utils.hasPackageTypePrefix(sinkPkgUrl)) {
- componentPackageFile = downloadPackageFile(sinkPkgUrl);
- } else {
- try {
- componentPackageFile = FunctionCommon.extractFileFromPkgURL(sinkPkgUrl);
- } catch (Exception e) {
- throw new IllegalArgumentException(
- String.format("Encountered error \"%s\" when getting %s package from %s",
- e.getMessage(), ComponentTypeUtils.toString(componentType), sinkPkgUrl));
- }
- }
- functionDetails = validateUpdateRequestParams(tenant, namespace, sinkName,
- mergedConfig, componentPackageFile);
-
- } else if (existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.FILE)
- || existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.HTTP)) {
- try {
- componentPackageFile = FunctionCommon
- .extractFileFromPkgURL(existingComponent.getPackageLocation().getPackagePath());
- } catch (Exception e) {
- throw new IllegalArgumentException(
- String.format("Encountered error \"%s\" when getting %s package from %s",
- e.getMessage(), ComponentTypeUtils.toString(componentType), sinkPkgUrl));
- }
- functionDetails = validateUpdateRequestParams(tenant, namespace, sinkName,
- mergedConfig, componentPackageFile);
- } else if (uploadedInputStream != null) {
-
- componentPackageFile = WorkerUtils.dumpToTmpFile(uploadedInputStream);
- functionDetails = validateUpdateRequestParams(tenant, namespace, sinkName,
- mergedConfig, componentPackageFile);
-
- } else if (existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.BUILTIN)) {
- functionDetails = validateUpdateRequestParams(tenant, namespace, sinkName,
- mergedConfig, componentPackageFile);
- if (!isFunctionCodeBuiltin(functionDetails)
- && (componentPackageFile == null || fileDetail == null)) {
+ componentPackageFile = getPackageFile(
+ sinkPkgUrl,
+ existingComponent.getPackageLocation().getPackagePath(),
+ uploadedInputStream);
+ functionDetails = validateUpdateRequestParams(tenant, namespace, sinkName,
+ mergedConfig, componentPackageFile);
+ if (existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.BUILTIN)
+ && !isFunctionCodeBuiltin(functionDetails)
+ && (componentPackageFile == null || fileDetail == null)) {
throw new IllegalArgumentException(
ComponentTypeUtils.toString(componentType) + " Package is not provided");
}
- } else {
- componentPackageFile = FunctionCommon.createPkgTempFile();
- componentPackageFile.deleteOnExit();
- if (worker().getWorkerConfig().isFunctionsWorkerEnablePackageManagement()) {
- worker().getBrokerAdmin().packages().download(
- existingComponent.getPackageLocation().getPackagePath(),
- componentPackageFile.getAbsolutePath());
- } else {
- WorkerUtils.downloadFromBookkeeper(worker().getDlogNamespace(), componentPackageFile,
- existingComponent.getPackageLocation().getPackagePath());
- }
- functionDetails = validateUpdateRequestParams(tenant, namespace, sinkName,
- mergedConfig, componentPackageFile);
- }
} catch (Exception e) {
log.error("Invalid update {} request @ /{}/{}/{}", ComponentTypeUtils.toString(componentType), tenant,
namespace, sinkName, e);
@@ -713,7 +658,8 @@ public class SinksImpl extends ComponentImpl implements Sinks<PulsarWorkerServic
final String namespace,
final String sinkName,
final SinkConfig sinkConfig,
- final File sinkPackageFile) throws IOException {
+ final File sinkPackageFile)
+ throws IOException, PulsarAdminException {
// The rest end points take precedence over whatever is there in sinkConfig
sinkConfig.setTenant(tenant);
@@ -721,7 +667,7 @@ public class SinksImpl extends ComponentImpl implements Sinks<PulsarWorkerServic
sinkConfig.setName(sinkName);
org.apache.pulsar.common.functions.Utils.inferMissingArguments(sinkConfig);
- ClassLoader classLoader = null;
+ ValidatableFunctionPackage connectorFunctionPackage = null;
// check if sink is builtin and extract classloader
if (!StringUtils.isEmpty(sinkConfig.getArchive())) {
String archive = sinkConfig.getArchive();
@@ -733,35 +679,39 @@ public class SinksImpl extends ComponentImpl implements Sinks<PulsarWorkerServic
if (connector == null) {
throw new IllegalArgumentException("Built-in sink is not available");
}
- classLoader = connector.getClassLoader();
+ connectorFunctionPackage = connector.getConnectorFunctionPackage();
}
}
- boolean shouldCloseClassLoader = false;
+ boolean shouldCloseFunctionPackage = false;
try {
// if sink is not builtin, attempt to extract classloader from package file if it exists
- if (classLoader == null && sinkPackageFile != null) {
- classLoader = getClassLoaderFromPackage(sinkConfig.getClassName(),
- sinkPackageFile, worker().getWorkerConfig().getNarExtractionDirectory());
- shouldCloseClassLoader = true;
+ WorkerConfig workerConfig = worker().getWorkerConfig();
+ if (connectorFunctionPackage == null && sinkPackageFile != null) {
+ connectorFunctionPackage =
+ new FunctionFilePackage(sinkPackageFile, workerConfig.getNarExtractionDirectory(),
+ workerConfig.getEnableClassloadingOfExternalFiles(), ConnectorDefinition.class);
+ shouldCloseFunctionPackage = true;
}
- if (classLoader == null) {
+ if (connectorFunctionPackage == null) {
throw new IllegalArgumentException("Sink package is not provided");
}
- SinkConfigUtils.ExtractedSinkDetails sinkDetails = SinkConfigUtils.validateAndExtractDetails(
- sinkConfig, classLoader, worker().getWorkerConfig().getValidateConnectorConfig());
+ SinkConfigUtils.ExtractedSinkDetails sinkDetails =
+ SinkConfigUtils.validateAndExtractDetails(sinkConfig, connectorFunctionPackage,
+ workerConfig.getValidateConnectorConfig());
+
return SinkConfigUtils.convert(sinkConfig, sinkDetails);
} finally {
- if (shouldCloseClassLoader) {
- ClassLoaderUtils.closeClassLoader(classLoader);
+ if (shouldCloseFunctionPackage && connectorFunctionPackage instanceof AutoCloseable) {
+ try {
+ ((AutoCloseable) connectorFunctionPackage).close();
+ } catch (Exception e) {
+ log.error("Failed to connector function file", e);
+ }
}
}
}
-
- private File downloadPackageFile(String packageName) throws IOException, PulsarAdminException {
- return FunctionsImpl.downloadPackageFile(worker(), packageName);
- }
}
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java
index 44c4115575a..59bc5888a34 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java
@@ -25,7 +25,6 @@ import static org.apache.pulsar.functions.utils.FunctionCommon.isFunctionCodeBui
import static org.apache.pulsar.functions.worker.rest.RestUtils.throwUnavailableException;
import com.google.protobuf.ByteString;
import java.io.File;
-import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.util.ArrayList;
@@ -47,19 +46,20 @@ import org.apache.pulsar.common.io.ConnectorDefinition;
import org.apache.pulsar.common.io.SourceConfig;
import org.apache.pulsar.common.policies.data.ExceptionInformation;
import org.apache.pulsar.common.policies.data.SourceStatus;
-import org.apache.pulsar.common.util.ClassLoaderUtils;
import org.apache.pulsar.common.util.RestException;
import org.apache.pulsar.functions.auth.FunctionAuthData;
import org.apache.pulsar.functions.instance.InstanceUtils;
import org.apache.pulsar.functions.proto.Function;
import org.apache.pulsar.functions.proto.InstanceCommunication;
import org.apache.pulsar.functions.utils.ComponentTypeUtils;
-import org.apache.pulsar.functions.utils.FunctionCommon;
+import org.apache.pulsar.functions.utils.FunctionFilePackage;
import org.apache.pulsar.functions.utils.FunctionMetaDataUtils;
import org.apache.pulsar.functions.utils.SourceConfigUtils;
+import org.apache.pulsar.functions.utils.ValidatableFunctionPackage;
import org.apache.pulsar.functions.utils.io.Connector;
import org.apache.pulsar.functions.worker.FunctionMetaDataManager;
import org.apache.pulsar.functions.worker.PulsarWorkerService;
+import org.apache.pulsar.functions.worker.WorkerConfig;
import org.apache.pulsar.functions.worker.WorkerUtils;
import org.apache.pulsar.functions.worker.service.api.Sources;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
@@ -143,21 +143,7 @@ public class SourcesImpl extends ComponentImpl implements Sources<PulsarWorkerSe
// validate parameters
try {
if (isPkgUrlProvided) {
- if (Utils.hasPackageTypePrefix(sourcePkgUrl)) {
- componentPackageFile = downloadPackageFile(sourcePkgUrl);
- } else {
- if (!Utils.isFunctionPackageUrlSupported(sourcePkgUrl)) {
- throw new IllegalArgumentException(
- "Function Package url is not valid. supported url (http/https/file)");
- }
- try {
- componentPackageFile = FunctionCommon.extractFileFromPkgURL(sourcePkgUrl);
- } catch (Exception e) {
- throw new IllegalArgumentException(
- String.format("Encountered error \"%s\" when getting %s package from %s",
- e.getMessage(), ComponentTypeUtils.toString(componentType), sourcePkgUrl));
- }
- }
+ componentPackageFile = getPackageFile(sourcePkgUrl);
functionDetails = validateUpdateRequestParams(tenant, namespace, sourceName,
sourceConfig, componentPackageFile);
} else {
@@ -305,71 +291,29 @@ public class SourcesImpl extends ComponentImpl implements Sources<PulsarWorkerSe
throw new RestException(Response.Status.BAD_REQUEST, e.getMessage());
}
- if (existingSourceConfig.equals(mergedConfig) && isBlank(sourcePkgUrl) && uploadedInputStream == null) {
+ if (existingSourceConfig.equals(mergedConfig) && isBlank(sourcePkgUrl) && uploadedInputStream == null
+ && (updateOptions == null || !updateOptions.isUpdateAuthData())) {
log.error("{}/{}/{} Update contains no changes", tenant, namespace, sourceName);
throw new RestException(Response.Status.BAD_REQUEST, "Update contains no change");
}
- Function.FunctionDetails functionDetails = null;
+ Function.FunctionDetails functionDetails;
File componentPackageFile = null;
try {
// validate parameters
try {
- if (isNotBlank(sourcePkgUrl)) {
- if (Utils.hasPackageTypePrefix(sourcePkgUrl)) {
- componentPackageFile = downloadPackageFile(sourcePkgUrl);
- } else {
- try {
- componentPackageFile = FunctionCommon.extractFileFromPkgURL(sourcePkgUrl);
- } catch (Exception e) {
- throw new IllegalArgumentException(
- String.format("Encountered error \"%s\" when getting %s package from %s",
- e.getMessage(), ComponentTypeUtils.toString(componentType), sourcePkgUrl));
- }
- }
- functionDetails = validateUpdateRequestParams(tenant, namespace, sourceName,
- mergedConfig, componentPackageFile);
-
- } else if (existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.FILE)
- || existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.HTTP)) {
- try {
- componentPackageFile = FunctionCommon
- .extractFileFromPkgURL(existingComponent.getPackageLocation().getPackagePath());
- } catch (Exception e) {
- throw new IllegalArgumentException(
- String.format("Encountered error \"%s\" when getting %s package from %s",
- e.getMessage(), ComponentTypeUtils.toString(componentType), sourcePkgUrl));
- }
- functionDetails = validateUpdateRequestParams(tenant, namespace, sourceName,
- mergedConfig, componentPackageFile);
- } else if (uploadedInputStream != null) {
-
- componentPackageFile = WorkerUtils.dumpToTmpFile(uploadedInputStream);
- functionDetails = validateUpdateRequestParams(tenant, namespace, sourceName,
- mergedConfig, componentPackageFile);
-
- } else if (existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.BUILTIN)) {
- functionDetails = validateUpdateRequestParams(tenant, namespace, sourceName,
- mergedConfig, componentPackageFile);
- if (!isFunctionCodeBuiltin(functionDetails)
- && (componentPackageFile == null || fileDetail == null)) {
- throw new IllegalArgumentException(
- ComponentTypeUtils.toString(componentType) + " Package is not provided");
- }
- } else {
- componentPackageFile = FunctionCommon.createPkgTempFile();
- componentPackageFile.deleteOnExit();
- if (worker().getWorkerConfig().isFunctionsWorkerEnablePackageManagement()) {
- worker().getBrokerAdmin().packages().download(
- existingComponent.getPackageLocation().getPackagePath(),
- componentPackageFile.getAbsolutePath());
- } else {
- WorkerUtils.downloadFromBookkeeper(worker().getDlogNamespace(), componentPackageFile,
- existingComponent.getPackageLocation().getPackagePath());
- }
- functionDetails = validateUpdateRequestParams(tenant, namespace, sourceName,
- mergedConfig, componentPackageFile);
+ componentPackageFile = getPackageFile(
+ sourcePkgUrl,
+ existingComponent.getPackageLocation().getPackagePath(),
+ uploadedInputStream);
+ functionDetails = validateUpdateRequestParams(tenant, namespace, sourceName,
+ mergedConfig, componentPackageFile);
+ if (existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.BUILTIN)
+ && !isFunctionCodeBuiltin(functionDetails)
+ && (componentPackageFile == null || fileDetail == null)) {
+ throw new IllegalArgumentException(ComponentTypeUtils.toString(componentType)
+ + " Package is not provided");
}
} catch (Exception e) {
log.error("Invalid update {} request @ /{}/{}/{}", ComponentTypeUtils.toString(componentType), tenant,
@@ -721,7 +665,7 @@ public class SourcesImpl extends ComponentImpl implements Sources<PulsarWorkerSe
sourceConfig.setName(sourceName);
org.apache.pulsar.common.functions.Utils.inferMissingArguments(sourceConfig);
- ClassLoader classLoader = null;
+ ValidatableFunctionPackage connectorFunctionPackage = null;
// check if source is builtin and extract classloader
if (!StringUtils.isEmpty(sourceConfig.getArchive())) {
String archive = sourceConfig.getArchive();
@@ -733,35 +677,38 @@ public class SourcesImpl extends ComponentImpl implements Sources<PulsarWorkerSe
if (connector == null) {
throw new IllegalArgumentException("Built-in source is not available");
}
- classLoader = connector.getClassLoader();
+ connectorFunctionPackage = connector.getConnectorFunctionPackage();
}
}
- boolean shouldCloseClassLoader = false;
+ boolean shouldCloseFunctionPackage = false;
try {
// if source is not builtin, attempt to extract classloader from package file if it exists
- if (classLoader == null && sourcePackageFile != null) {
- classLoader = getClassLoaderFromPackage(sourceConfig.getClassName(),
- sourcePackageFile, worker().getWorkerConfig().getNarExtractionDirectory());
- shouldCloseClassLoader = true;
+ WorkerConfig workerConfig = worker().getWorkerConfig();
+ if (connectorFunctionPackage == null && sourcePackageFile != null) {
+ connectorFunctionPackage =
+ new FunctionFilePackage(sourcePackageFile, workerConfig.getNarExtractionDirectory(),
+ workerConfig.getEnableClassloadingOfExternalFiles(), ConnectorDefinition.class);
+ shouldCloseFunctionPackage = true;
}
- if (classLoader == null) {
+ if (connectorFunctionPackage == null) {
throw new IllegalArgumentException("Source package is not provided");
}
SourceConfigUtils.ExtractedSourceDetails sourceDetails =
SourceConfigUtils.validateAndExtractDetails(
- sourceConfig, classLoader, worker().getWorkerConfig().getValidateConnectorConfig());
+ sourceConfig, connectorFunctionPackage,
+ workerConfig.getValidateConnectorConfig());
return SourceConfigUtils.convert(sourceConfig, sourceDetails);
} finally {
- if (shouldCloseClassLoader) {
- ClassLoaderUtils.closeClassLoader(classLoader);
+ if (shouldCloseFunctionPackage && connectorFunctionPackage instanceof AutoCloseable) {
+ try {
+ ((AutoCloseable) connectorFunctionPackage).close();
+ } catch (Exception e) {
+ log.error("Failed to connector function file", e);
+ }
}
}
}
-
- private File downloadPackageFile(String packageName) throws IOException, PulsarAdminException {
- return FunctionsImpl.downloadPackageFile(worker(), packageName);
- }
}
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionAssignmentTailerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionAssignmentTailerTest.java
index a64d451628d..459bf243c1e 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionAssignmentTailerTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionAssignmentTailerTest.java
@@ -145,6 +145,9 @@ public class FunctionAssignmentTailerTest {
// test new assignment add functions
FunctionRuntimeManager functionRuntimeManager = mock(FunctionRuntimeManager.class);
+ // trigger an error to be thrown
+ doThrow(new RuntimeException("test")).when(functionRuntimeManager).processAssignmentMessage(any());
+
FunctionAssignmentTailer functionAssignmentTailer =
spy(new FunctionAssignmentTailer(functionRuntimeManager, readerBuilder, workerConfig, errorNotifier));
@@ -157,9 +160,6 @@ public class FunctionAssignmentTailerTest {
verify(errorNotifier, times(0)).triggerError(any());
- // trigger an error to be thrown
- doThrow(new RuntimeException("test")).when(functionRuntimeManager).processAssignmentMessage(any());
-
messageList.add(message2);
try {
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java
index 7443e1e62d8..d8902c00304 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java
@@ -381,6 +381,6 @@ public class FunctionsImplTest {
public static Function.FunctionDetails createDefaultFunctionDetails() {
FunctionConfig functionConfig = createDefaultFunctionConfig();
- return FunctionConfigUtils.convert(functionConfig, (ClassLoader) null);
+ return FunctionConfigUtils.convert(functionConfig);
}
}
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java
index b50e734e0d5..198baa6a199 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java
@@ -18,1125 +18,82 @@
*/
package org.apache.pulsar.functions.worker.rest.api.v2;
-
-import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import static org.apache.pulsar.functions.utils.FunctionCommon.mergeJson;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
-import com.google.common.collect.Lists;
import com.google.gson.Gson;
-import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.HashMap;
-import java.util.LinkedList;
import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.function.Consumer;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
-import org.apache.distributedlog.api.namespace.Namespace;
-import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.core.config.Configurator;
import org.apache.pulsar.broker.authentication.AuthenticationParameters;
-import org.apache.pulsar.client.admin.Functions;
-import org.apache.pulsar.client.admin.Namespaces;
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.client.admin.Tenants;
import org.apache.pulsar.common.functions.FunctionConfig;
-import org.apache.pulsar.common.policies.data.TenantInfoImpl;
+import org.apache.pulsar.common.functions.UpdateOptionsImpl;
import org.apache.pulsar.common.util.RestException;
-import org.apache.pulsar.functions.api.Context;
-import org.apache.pulsar.functions.api.Function;
-import org.apache.pulsar.functions.instance.InstanceUtils;
-import org.apache.pulsar.functions.proto.Function.FunctionDetails;
-import org.apache.pulsar.functions.proto.Function.FunctionMetaData;
-import org.apache.pulsar.functions.proto.Function.PackageLocationMetaData;
-import org.apache.pulsar.functions.proto.Function.ProcessingGuarantees;
-import org.apache.pulsar.functions.proto.Function.SinkSpec;
-import org.apache.pulsar.functions.proto.Function.SourceSpec;
-import org.apache.pulsar.functions.proto.Function.SubscriptionType;
-import org.apache.pulsar.functions.runtime.RuntimeFactory;
-import org.apache.pulsar.functions.source.TopicSchema;
-import org.apache.pulsar.functions.utils.FunctionCommon;
+import org.apache.pulsar.functions.proto.Function;
import org.apache.pulsar.functions.utils.FunctionConfigUtils;
-import org.apache.pulsar.functions.worker.FunctionMetaDataManager;
-import org.apache.pulsar.functions.worker.FunctionRuntimeManager;
-import org.apache.pulsar.functions.worker.LeaderService;
-import org.apache.pulsar.functions.worker.PulsarWorkerService;
-import org.apache.pulsar.functions.worker.WorkerConfig;
-import org.apache.pulsar.functions.worker.WorkerUtils;
-import org.apache.pulsar.functions.worker.rest.api.FunctionsImpl;
import org.apache.pulsar.functions.worker.rest.api.FunctionsImplV2;
-import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory;
+import org.apache.pulsar.functions.worker.rest.api.v3.AbstractFunctionApiResourceTest;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
-import org.mockito.MockedStatic;
-import org.mockito.Mockito;
-import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
-/**
- * Unit test of {@link FunctionsApiV2Resource}.
- */
-public class FunctionApiV2ResourceTest {
-
- private static final class TestFunction implements Function<String, String> {
-
- @Override
- public String process(String input, Context context) {
- return input;
- }
- }
-
- private static final String tenant = "test-tenant";
- private static final String namespace = "test-namespace";
- private static final String function = "test-function";
- private static final String outputTopic = "test-output-topic";
- private static final String outputSerdeClassName = TopicSchema.DEFAULT_SERDE;
- private static final String className = TestFunction.class.getName();
- private SubscriptionType subscriptionType = SubscriptionType.FAILOVER;
- private static final Map<String, String> topicsToSerDeClassName = new HashMap<>();
- static {
- topicsToSerDeClassName.put("persistent://public/default/test_src", TopicSchema.DEFAULT_SERDE);
- }
- private static final int parallelism = 1;
-
- private PulsarWorkerService mockedWorkerService;
- private PulsarAdmin mockedPulsarAdmin;
- private Tenants mockedTenants;
- private Namespaces mockedNamespaces;
- private Functions mockedFunctions;
- private TenantInfoImpl mockedTenantInfo;
- private List<String> namespaceList = new LinkedList<>();
- private FunctionMetaDataManager mockedManager;
- private FunctionRuntimeManager mockedFunctionRunTimeManager;
- private RuntimeFactory mockedRuntimeFactory;
- private Namespace mockedNamespace;
+public class FunctionApiV2ResourceTest extends AbstractFunctionApiResourceTest {
private FunctionsImplV2 resource;
- private InputStream mockedInputStream;
- private FormDataContentDisposition mockedFormData;
- private FunctionMetaData mockedFunctionMetadata;
- private LeaderService mockedLeaderService;
- private PulsarFunctionTestTemporaryDirectory tempDirectory;
- private static Map<String, MockedStatic> mockStaticContexts = new HashMap<>();
-
- @BeforeMethod
- public void setup() throws Exception {
- this.mockedManager = mock(FunctionMetaDataManager.class);
- this.mockedFunctionRunTimeManager = mock(FunctionRuntimeManager.class);
- this.mockedTenantInfo = mock(TenantInfoImpl.class);
- this.mockedRuntimeFactory = mock(RuntimeFactory.class);
- this.mockedInputStream = mock(InputStream.class);
- this.mockedNamespace = mock(Namespace.class);
- this.mockedFormData = mock(FormDataContentDisposition.class);
- when(mockedFormData.getFileName()).thenReturn("test");
- this.mockedPulsarAdmin = mock(PulsarAdmin.class);
- this.mockedTenants = mock(Tenants.class);
- this.mockedNamespaces = mock(Namespaces.class);
- this.mockedFunctions = mock(Functions.class);
- this.mockedLeaderService = mock(LeaderService.class);
- this.mockedFunctionMetadata = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build();
- namespaceList.add(tenant + "/" + namespace);
-
- this.mockedWorkerService = mock(PulsarWorkerService.class);
- when(mockedWorkerService.getFunctionMetaDataManager()).thenReturn(mockedManager);
- when(mockedWorkerService.getLeaderService()).thenReturn(mockedLeaderService);
- when(mockedWorkerService.getFunctionRuntimeManager()).thenReturn(mockedFunctionRunTimeManager);
- when(mockedFunctionRunTimeManager.getRuntimeFactory()).thenReturn(mockedRuntimeFactory);
- when(mockedWorkerService.getDlogNamespace()).thenReturn(mockedNamespace);
- when(mockedWorkerService.isInitialized()).thenReturn(true);
- when(mockedWorkerService.getBrokerAdmin()).thenReturn(mockedPulsarAdmin);
- when(mockedWorkerService.getFunctionAdmin()).thenReturn(mockedPulsarAdmin);
- when(mockedPulsarAdmin.tenants()).thenReturn(mockedTenants);
- when(mockedPulsarAdmin.namespaces()).thenReturn(mockedNamespaces);
- when(mockedPulsarAdmin.functions()).thenReturn(mockedFunctions);
- when(mockedTenants.getTenantInfo(any())).thenReturn(mockedTenantInfo);
- when(mockedNamespaces.getNamespaces(any())).thenReturn(namespaceList);
- when(mockedLeaderService.isLeader()).thenReturn(true);
- when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetadata);
-
- // worker config
- WorkerConfig workerConfig = new WorkerConfig()
- .setWorkerId("test")
- .setWorkerPort(8080)
- .setFunctionMetadataTopicName("pulsar/functions")
- .setNumFunctionPackageReplicas(3)
- .setPulsarServiceUrl("pulsar://localhost:6650/");
- tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName());
- tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig);
- when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig);
-
- FunctionsImpl functions = spy(new FunctionsImpl(() -> mockedWorkerService));
-
- this.resource = spy(new FunctionsImplV2(functions));
-
- }
-
- @AfterMethod(alwaysRun = true)
- public void cleanup() {
- if (tempDirectory != null) {
- tempDirectory.delete();
- }
- mockStaticContexts.values().forEach(MockedStatic::close);
- mockStaticContexts.clear();
- }
-
- private <T> void mockStatic(Class<T> classStatic, Consumer<MockedStatic<T>> consumer) {
- final MockedStatic<T> mockedStatic =
- mockStaticContexts.computeIfAbsent(classStatic.getName(), name -> Mockito.mockStatic(classStatic));
- consumer.accept(mockedStatic);
- }
-
- private void mockWorkerUtils() {
- mockStatic(WorkerUtils.class, ctx -> {
- ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod();
- });
- }
-
- private void mockWorkerUtils(Consumer<MockedStatic<WorkerUtils>> consumer) {
- mockStatic(WorkerUtils.class, ctx -> {
- ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod();
- if (consumer != null) {
- consumer.accept(ctx);
- }
- });
- }
-
- private void mockInstanceUtils() {
- mockStatic(InstanceUtils.class, ctx -> {
- ctx.when(() -> InstanceUtils.calculateSubjectType(any()))
- .thenReturn(FunctionDetails.ComponentType.FUNCTION);
- });
- }
-
- //
- // Register Functions
- //
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided")
- public void testRegisterFunctionMissingTenant() {
- try {
- testRegisterFunctionMissingArguments(
- null,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided")
- public void testRegisterFunctionMissingNamespace() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- null,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided")
- public void testRegisterFunctionMissingFunctionName() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- null,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package is not provided")
- public void testRegisterFunctionMissingPackage() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- null,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "No input topic\\(s\\) specified for the function")
- public void testRegisterFunctionMissingInputTopics() throws Exception {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- mockedInputStream,
- null,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function Package is not provided")
- public void testRegisterFunctionMissingPackageDetails() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- null,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package does not have"
- + " the correct format. Pulsar cannot determine if the package is a NAR package or JAR package. Function "
- + "classname is not provided and attempts to load it as a NAR package produced the following error.*")
- public void testRegisterFunctionMissingClassName() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- null,
- parallelism,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function class UnknownClass must be in class path")
- public void testRegisterFunctionWrongClassName() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- "UnknownClass",
- parallelism,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function parallelism must be a positive number")
- public void testRegisterFunctionWrongParallelism() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- -2,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class,
- expectedExceptionsMessageRegExp = "Output topic persistent://public/default/test_src is also being used as an input topic \\(topics must be one or the other\\)")
- public void testRegisterFunctionSameInputOutput() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- topicsToSerDeClassName.keySet().iterator().next(),
- outputSerdeClassName,
- className,
- parallelism,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Output topic " + function +
- "-output-topic/test:" + " is invalid")
- public void testRegisterFunctionWrongOutputTopic() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- function + "-output-topic/test:",
- outputSerdeClassName,
- className,
- parallelism,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Encountered error .*. when getting Function package from .*")
- public void testRegisterFunctionHttpUrl() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- null,
- topicsToSerDeClassName,
- null,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- "http://localhost:1234/test");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- private void testRegisterFunctionMissingArguments(
- String tenant,
- String namespace,
- String function,
- InputStream inputStream,
- Map<String, String> topicsToSerDeClassName,
- FormDataContentDisposition details,
- String outputTopic,
- String outputSerdeClassName,
- String className,
- Integer parallelism,
- String functionPkgUrl) {
- FunctionConfig functionConfig = new FunctionConfig();
- if (tenant != null) {
- functionConfig.setTenant(tenant);
- }
- if (namespace != null) {
- functionConfig.setNamespace(namespace);
- }
- if (function != null) {
- functionConfig.setName(function);
- }
- if (topicsToSerDeClassName != null) {
- functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
- }
- if (outputTopic != null) {
- functionConfig.setOutput(outputTopic);
- }
- if (outputSerdeClassName != null) {
- functionConfig.setOutputSerdeClassName(outputSerdeClassName);
- }
- if (className != null) {
- functionConfig.setClassName(className);
- }
- if (parallelism != null) {
- functionConfig.setParallelism(parallelism);
- }
- functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
-
- mockWorkerUtils(ctx -> {
- ctx.when(() -> WorkerUtils.uploadFileToBookkeeper(
- anyString(),
- any(File.class),
- any(Namespace.class)))
- .thenCallRealMethod();
- });
-
- try {
- resource.registerFunction(
- tenant,
- namespace,
- function,
- inputStream,
- details,
- functionPkgUrl,
- JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
- AuthenticationParameters.builder().build());
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException(e);
- }
-
- }
-
- private void registerDefaultFunction() {
- FunctionConfig functionConfig = createDefaultFunctionConfig();
- try {
- resource.registerFunction(
- tenant,
- namespace,
- function,
- mockedInputStream,
- mockedFormData,
- null,
- JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
- AuthenticationParameters.builder().build());
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function already exists")
- public void testRegisterExistedFunction() {
- try {
- Configurator.setRootLevel(Level.DEBUG);
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
- registerDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure")
- public void testRegisterFunctionUploadFailure() throws Exception {
- try {
- mockWorkerUtils(ctx -> {
- ctx.when(() -> WorkerUtils.uploadFileToBookkeeper(
- anyString(),
- any(File.class),
- any(Namespace.class)))
- .thenThrow(new IOException("upload failure"));
- });
-
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
-
- registerDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR);
- throw re;
- }
- }
-
- @Test
- public void testRegisterFunctionSuccess() throws Exception {
- try {
- try (MockedStatic mocked = Mockito.mockStatic(WorkerUtils.class)) {
- mocked.when(() -> WorkerUtils.uploadToBookKeeper(
- any(Namespace.class),
- any(InputStream.class),
- anyString())).thenAnswer((i) -> null);
- mocked.when(() -> WorkerUtils.dumpToTmpFile(any())).thenAnswer(i ->
- {
- try {
- File tmpFile = FunctionCommon.createPkgTempFile();
- tmpFile.deleteOnExit();
- Files.copy((InputStream) i.getArguments()[0], tmpFile.toPath(), REPLACE_EXISTING);
- return tmpFile;
- } catch (IOException e) {
- throw new RuntimeException("Cannot create a temporary file", e);
- }
-
- }
- );
- WorkerUtils.uploadToBookKeeper(null, null, null);
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
-
- registerDefaultFunction();
- }
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace does not exist")
- public void testRegisterFunctionNonExistingNamespace() {
- try {
- this.namespaceList.clear();
- registerDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant does not exist")
- public void testRegisterFunctionNonexistantTenant() throws Exception {
- try {
- when(mockedTenants.getTenantInfo(any())).thenThrow(PulsarAdminException.NotFoundException.class);
- registerDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register")
- public void testRegisterFunctionFailure() throws Exception {
- try {
- mockWorkerUtils();
-
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
- doThrow(new IllegalArgumentException("function failed to register"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean());
-
- registerDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
-
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registration interrupted")
- public void testRegisterFunctionInterrupted() throws Exception {
- try {
- mockWorkerUtils();
-
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
-
- doThrow(new IllegalStateException("Function registration interrupted"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean());
- registerDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR);
- throw re;
- }
- }
-
- //
- // Update Functions
- //
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided")
- public void testUpdateFunctionMissingTenant() throws Exception {
- try {
- testUpdateFunctionMissingArguments(
- null,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- "Tenant is not provided");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided")
- public void testUpdateFunctionMissingNamespace() throws Exception {
- try {
- testUpdateFunctionMissingArguments(
- tenant,
- null,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- "Namespace is not provided");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided")
- public void testUpdateFunctionMissingFunctionName() throws Exception {
- try {
- testUpdateFunctionMissingArguments(
- tenant,
- namespace,
- null,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- "Function name is not provided");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change")
- public void testUpdateFunctionMissingPackage() throws Exception {
- try {
- mockWorkerUtils();
- testUpdateFunctionMissingArguments(
- tenant,
- namespace,
- function,
- null,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- "Update contains no change");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change")
- public void testUpdateFunctionMissingInputTopic() throws Exception {
- try {
- mockWorkerUtils();
-
- testUpdateFunctionMissingArguments(
- tenant,
- namespace,
- function,
- null,
- null,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- "Update contains no change");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change")
- public void testUpdateFunctionMissingClassName() throws Exception {
- try {
- mockWorkerUtils();
-
- testUpdateFunctionMissingArguments(
- tenant,
- namespace,
- function,
- null,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- null,
- parallelism,
- "Update contains no change");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test
- public void testUpdateFunctionChangedParallelism() throws Exception {
- try {
- mockWorkerUtils();
-
- testUpdateFunctionMissingArguments(
- tenant,
- namespace,
- function,
- null,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- null,
- parallelism + 1,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
+ @Override
+ protected void doSetup() {
+ super.doSetup();
+ this.resource = spy(new FunctionsImplV2(() -> mockedWorkerService));
}
- @Test
- public void testUpdateFunctionChangedInputs() throws Exception {
- mockWorkerUtils();
-
- testUpdateFunctionMissingArguments(
+ protected void registerFunction(String tenant, String namespace, String function, InputStream inputStream,
+ FormDataContentDisposition details, String functionPkgUrl,
+ FunctionConfig functionConfig) throws IOException {
+ resource.registerFunction(
tenant,
namespace,
function,
- null,
- topicsToSerDeClassName,
- mockedFormData,
- "DifferentOutput",
- outputSerdeClassName,
- null,
- parallelism,
+ inputStream,
+ details,
+ functionPkgUrl,
+ JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig)),
null);
}
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Input Topics cannot be altered")
- public void testUpdateFunctionChangedOutput() throws Exception {
- try {
- mockWorkerUtils();
-
- Map<String, String> someOtherInput = new HashMap<>();
- someOtherInput.put("DifferentTopic", TopicSchema.DEFAULT_SERDE);
- testUpdateFunctionMissingArguments(
- tenant,
- namespace,
- function,
- null,
- someOtherInput,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- null,
- parallelism,
- "Input Topics cannot be altered");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- private void testUpdateFunctionMissingArguments(
- String tenant,
- String namespace,
- String function,
- InputStream inputStream,
- Map<String, String> topicsToSerDeClassName,
- FormDataContentDisposition details,
- String outputTopic,
- String outputSerdeClassName,
- String className,
- Integer parallelism,
- String expectedError) throws Exception {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- FunctionConfig functionConfig = new FunctionConfig();
- if (tenant != null) {
- functionConfig.setTenant(tenant);
- }
- if (namespace != null) {
- functionConfig.setNamespace(namespace);
- }
- if (function != null) {
- functionConfig.setName(function);
- }
- if (topicsToSerDeClassName != null) {
- functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
- }
- if (outputTopic != null) {
- functionConfig.setOutput(outputTopic);
- }
- if (outputSerdeClassName != null) {
- functionConfig.setOutputSerdeClassName(outputSerdeClassName);
- }
- if (className != null) {
- functionConfig.setClassName(className);
- }
- if (parallelism != null) {
- functionConfig.setParallelism(parallelism);
- }
- functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
-
- if (expectedError != null) {
- doThrow(new IllegalArgumentException(expectedError))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean());
- }
-
- try {
- resource.updateFunction(
- tenant,
- namespace,
- function,
- inputStream,
- details,
- null,
- JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
- AuthenticationParameters.builder().build());
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException(e);
- }
-
- }
-
- private void updateDefaultFunction() {
- FunctionConfig functionConfig = new FunctionConfig();
- functionConfig.setTenant(tenant);
- functionConfig.setNamespace(namespace);
- functionConfig.setName(function);
- functionConfig.setClassName(className);
- functionConfig.setParallelism(parallelism);
- functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
- functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
- functionConfig.setOutput(outputTopic);
- functionConfig.setOutputSerdeClassName(outputSerdeClassName);
-
- try {
- resource.updateFunction(
- tenant,
- namespace,
- function,
- mockedInputStream,
- mockedFormData,
- null,
- JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
- AuthenticationParameters.builder().build());
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist")
- public void testUpdateNotExistedFunction() {
- try {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
- updateDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure")
- public void testUpdateFunctionUploadFailure() throws Exception {
- try {
- mockWorkerUtils(ctx -> {
- ctx.when(() -> WorkerUtils.uploadFileToBookkeeper(
- anyString(),
- any(File.class),
- any(Namespace.class)))
- .thenThrow(new IOException("upload failure"));
- });
-
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- updateDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR);
- throw re;
- }
- }
-
- @Test
- public void testUpdateFunctionSuccess() throws Exception {
- mockWorkerUtils();
-
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- updateDefaultFunction();
- }
-
- @Test
- public void testUpdateFunctionWithUrl() throws Exception {
- Configurator.setRootLevel(Level.DEBUG);
-
- URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
- File file = Paths.get(fileUrl.toURI()).toFile();
- String fileLocation = file.getAbsolutePath().replace('\\', '/');
- String filePackageUrl = "file:///" + fileLocation;
-
- FunctionConfig functionConfig = new FunctionConfig();
- functionConfig.setOutput(outputTopic);
- functionConfig.setOutputSerdeClassName(outputSerdeClassName);
- functionConfig.setTenant(tenant);
- functionConfig.setNamespace(namespace);
- functionConfig.setName(function);
- functionConfig.setClassName(className);
- functionConfig.setParallelism(parallelism);
- functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
- functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
-
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- try {
- resource.updateFunction(
- tenant,
- namespace,
- function,
- null,
- null,
- filePackageUrl,
- JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
- AuthenticationParameters.builder().build());
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException(e);
- }
-
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register")
- public void testUpdateFunctionFailure() throws Exception {
- try {
- mockWorkerUtils();
-
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- doThrow(new IllegalArgumentException("function failed to register"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean());
-
- updateDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registeration interrupted")
- public void testUpdateFunctionInterrupted() throws Exception {
- try {
- mockWorkerUtils();
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- doThrow(new IllegalStateException("Function registeration interrupted"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean());
-
- updateDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR);
- throw re;
- }
- }
-
- //
- // deregister function
- //
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided")
- public void testDeregisterFunctionMissingTenant() {
- try {
-
- testDeregisterFunctionMissingArguments(
- null,
- namespace,
- function
- );
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
+ protected void updateFunction(String tenant,
+ String namespace,
+ String functionName,
+ InputStream uploadedInputStream,
+ FormDataContentDisposition fileDetail,
+ String functionPkgUrl,
+ FunctionConfig functionConfig,
+ AuthenticationParameters authParams,
+ UpdateOptionsImpl updateOptions) throws IOException {
+ resource.updateFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail, functionPkgUrl,
+ JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig)), authParams);
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided")
- public void testDeregisterFunctionMissingNamespace() {
- try {
- testDeregisterFunctionMissingArguments(
- tenant,
- null,
- function
- );
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
+ protected File downloadFunction(final String path, final AuthenticationParameters authParams)
+ throws IOException {
+ Response response = resource.downloadFunction(path, authParams);
+ StreamingOutput streamingOutput = readEntity(response, StreamingOutput.class);
+ File pkgFile = File.createTempFile("testpkg", "nar");
+ try (OutputStream output = new FileOutputStream(pkgFile)) {
+ streamingOutput.write(output);
}
+ return pkgFile;
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided")
- public void testDeregisterFunctionMissingFunctionName() {
- try {
- testDeregisterFunctionMissingArguments(
- tenant,
- namespace,
- null
- );
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
+ private <T> T readEntity(Response response, Class<T> clazz) {
+ return clazz.cast(response.getEntity());
}
- private void testDeregisterFunctionMissingArguments(
+ protected void testDeregisterFunctionMissingArguments(
String tenant,
String namespace,
String function
@@ -1145,112 +102,18 @@ public class FunctionApiV2ResourceTest {
tenant,
namespace,
function,
- AuthenticationParameters.builder().build());
+ null);
}
- private void deregisterDefaultFunction() {
+ protected void deregisterDefaultFunction() {
resource.deregisterFunction(
tenant,
namespace,
function,
- AuthenticationParameters.builder().build());
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist")
- public void testDeregisterNotExistedFunction() {
- try {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
- deregisterDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.NOT_FOUND);
- throw re;
- }
- }
-
- @Test
- public void testDeregisterFunctionSuccess() {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- deregisterDefaultFunction();
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to deregister")
- public void testDeregisterFunctionFailure() throws Exception {
- try {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- doThrow(new IllegalArgumentException("function failed to deregister"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean());
-
- deregisterDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function deregisteration interrupted")
- public void testDeregisterFunctionInterrupted() throws Exception {
- try {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- doThrow(new IllegalStateException("Function deregisteration interrupted"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean());
-
- deregisterDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR);
- throw re;
- }
- }
-
- //
- // Get Function Info
- //
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided")
- public void testGetFunctionMissingTenant() throws IOException {
- try {
- testGetFunctionMissingArguments(
- null,
- namespace,
- function
- );
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided")
- public void testGetFunctionMissingNamespace() throws IOException {
- try {
- testGetFunctionMissingArguments(
- tenant,
- null,
- function
- );
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided")
- public void testGetFunctionMissingFunctionName() throws IOException {
- try {
- testGetFunctionMissingArguments(
- tenant,
- namespace,
- null
- );
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
+ null);
}
- private void testGetFunctionMissingArguments(
+ protected void testGetFunctionMissingArguments(
String tenant,
String namespace,
String function
@@ -1258,20 +121,36 @@ public class FunctionApiV2ResourceTest {
resource.getFunctionInfo(
tenant,
namespace,
- function,
- AuthenticationParameters.builder().build()
+ function, null
+ );
+ }
+
+ protected void testListFunctionsMissingArguments(
+ String tenant,
+ String namespace
+ ) {
+ resource.listFunctions(
+ tenant,
+ namespace, null
);
}
- private FunctionDetails getDefaultFunctionInfo() throws IOException {
+ protected List<String> listDefaultFunctions() {
+ return new Gson().fromJson(readEntity(resource.listFunctions(
+ tenant,
+ namespace, null
+ ), String.class), List.class);
+ }
+
+ private Function.FunctionDetails getDefaultFunctionInfo() throws IOException {
String json = (String) resource.getFunctionInfo(
tenant,
namespace,
function,
AuthenticationParameters.builder().build()
).getEntity();
- FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
+ Function.FunctionDetails.Builder functionDetailsBuilder = Function.FunctionDetails.newBuilder();
mergeJson(json, functionDetailsBuilder);
return functionDetailsBuilder.build();
}
@@ -1292,218 +171,31 @@ public class FunctionApiV2ResourceTest {
mockInstanceUtils();
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
- SinkSpec sinkSpec = SinkSpec.newBuilder()
+ Function.SinkSpec sinkSpec = Function.SinkSpec.newBuilder()
.setTopic(outputTopic)
.setSerDeClassName(outputSerdeClassName).build();
- FunctionDetails functionDetails = FunctionDetails.newBuilder()
+ Function.FunctionDetails functionDetails = Function.FunctionDetails.newBuilder()
.setClassName(className)
.setSink(sinkSpec)
.setName(function)
.setNamespace(namespace)
- .setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE)
+ .setProcessingGuarantees(Function.ProcessingGuarantees.ATMOST_ONCE)
.setAutoAck(true)
.setTenant(tenant)
.setParallelism(parallelism)
- .setSource(SourceSpec.newBuilder().setSubscriptionType(subscriptionType)
+ .setSource(Function.SourceSpec.newBuilder().setSubscriptionType(subscriptionType)
.putAllTopicsToSerDeClassName(topicsToSerDeClassName)).build();
- FunctionMetaData metaData = FunctionMetaData.newBuilder()
+ Function.FunctionMetaData metaData = Function.FunctionMetaData.newBuilder()
.setCreateTime(System.currentTimeMillis())
.setFunctionDetails(functionDetails)
- .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("/path/to/package"))
+ .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/path/to/package"))
.setVersion(1234)
.build();
when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData);
- FunctionDetails actual = getDefaultFunctionInfo();
+ Function.FunctionDetails actual = getDefaultFunctionInfo();
assertEquals(
functionDetails,
actual);
}
-
- //
- // List Functions
- //
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided")
- public void testListFunctionsMissingTenant() {
- try {
- testListFunctionsMissingArguments(
- null,
- namespace
- );
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided")
- public void testListFunctionsMissingNamespace() {
- try {
- testListFunctionsMissingArguments(
- tenant,
- null
- );
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- private void testListFunctionsMissingArguments(
- String tenant,
- String namespace
- ) {
- resource.listFunctions(
- tenant,
- namespace,
- AuthenticationParameters.builder().build()
- );
-
- }
-
- private List<String> listDefaultFunctions() {
- return new Gson().fromJson((String) resource.listFunctions(
- tenant,
- namespace,
- AuthenticationParameters.builder().build()
- ).getEntity(), List.class);
- }
-
- @Test
- public void testListFunctionsSuccess() {
- mockInstanceUtils();
- final List<String> functions = Lists.newArrayList("test-1", "test-2");
- final List<FunctionMetaData> metaDataList = new LinkedList<>();
- FunctionMetaData functionMetaData1 = FunctionMetaData.newBuilder().setFunctionDetails(
- FunctionDetails.newBuilder().setName("test-1").build()
- ).build();
- FunctionMetaData functionMetaData2 = FunctionMetaData.newBuilder().setFunctionDetails(
- FunctionDetails.newBuilder().setName("test-2").build()
- ).build();
- metaDataList.add(functionMetaData1);
- metaDataList.add(functionMetaData2);
- when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(metaDataList);
-
- List<String> functionList = listDefaultFunctions();
- assertEquals(functions, functionList);
- }
-
- @Test
- public void testDownloadFunctionHttpUrl() throws Exception {
- String jarHttpUrl =
- "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar";
- String testDir = FunctionApiV2ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath();
- FunctionsImplV2 function = new FunctionsImplV2(() -> mockedWorkerService);
- StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction(jarHttpUrl,
- AuthenticationParameters.builder().build()).getEntity();
- File pkgFile = new File(testDir, UUID.randomUUID().toString());
- OutputStream output = new FileOutputStream(pkgFile);
- streamOutput.write(output);
- Assert.assertTrue(pkgFile.exists());
- if (pkgFile.exists()) {
- pkgFile.delete();
- }
- }
-
- @Test
- public void testDownloadFunctionFile() throws Exception {
- URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
- File file = Paths.get(fileUrl.toURI()).toFile();
- String fileLocation = file.getAbsolutePath().replace('\\', '/');
- String testDir = FunctionApiV2ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath();
- FunctionsImplV2 function = new FunctionsImplV2(() -> mockedWorkerService);
- StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction("file:///" + fileLocation,
- AuthenticationParameters.builder().build()).getEntity();
- File pkgFile = new File(testDir, UUID.randomUUID().toString());
- OutputStream output = new FileOutputStream(pkgFile);
- streamOutput.write(output);
- Assert.assertTrue(pkgFile.exists());
- if (pkgFile.exists()) {
- pkgFile.delete();
- }
- }
-
- @Test
- public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception {
- Configurator.setRootLevel(Level.DEBUG);
-
- URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
- File file = Paths.get(fileUrl.toURI()).toFile();
- String fileLocation = file.getAbsolutePath().replace('\\', '/');
- String filePackageUrl = "file:///" + fileLocation;
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
-
- FunctionConfig functionConfig = new FunctionConfig();
- functionConfig.setTenant(tenant);
- functionConfig.setNamespace(namespace);
- functionConfig.setName(function);
- functionConfig.setClassName(className);
- functionConfig.setParallelism(parallelism);
- functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
- functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
- functionConfig.setOutput(outputTopic);
- functionConfig.setOutputSerdeClassName(outputSerdeClassName);
- try {
- resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl,
- JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
- AuthenticationParameters.builder().build());
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException(e);
- }
-
- }
-
- @Test
- public void testRegisterFunctionWithConflictingFields() throws Exception {
- Configurator.setRootLevel(Level.DEBUG);
- String actualTenant = "DIFFERENT_TENANT";
- String actualNamespace = "DIFFERENT_NAMESPACE";
- String actualName = "DIFFERENT_NAME";
- this.namespaceList.add(actualTenant + "/" + actualNamespace);
-
- URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
- File file = Paths.get(fileUrl.toURI()).toFile();
- String fileLocation = file.getAbsolutePath().replace('\\', '/');
- String filePackageUrl = "file:///" + fileLocation;
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
- when(mockedManager.containsFunction(eq(actualTenant), eq(actualNamespace), eq(actualName))).thenReturn(false);
-
- FunctionConfig functionConfig = new FunctionConfig();
- functionConfig.setTenant(tenant);
- functionConfig.setNamespace(namespace);
- functionConfig.setName(function);
- functionConfig.setClassName(className);
- functionConfig.setParallelism(parallelism);
- functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
- functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
- functionConfig.setOutput(outputTopic);
- functionConfig.setOutputSerdeClassName(outputSerdeClassName);
- try {
- resource.registerFunction(actualTenant, actualNamespace, actualName, null, null, filePackageUrl,
- JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
- AuthenticationParameters.builder().build());
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException(e);
- }
- }
-
- public static FunctionConfig createDefaultFunctionConfig() {
- FunctionConfig functionConfig = new FunctionConfig();
- functionConfig.setTenant(tenant);
- functionConfig.setNamespace(namespace);
- functionConfig.setName(function);
- functionConfig.setClassName(className);
- functionConfig.setParallelism(parallelism);
- functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
- functionConfig.setOutput(outputTopic);
- functionConfig.setOutputSerdeClassName(outputSerdeClassName);
- functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
- return functionConfig;
- }
-
- public static FunctionDetails createDefaultFunctionDetails() {
- FunctionConfig functionConfig = createDefaultFunctionConfig();
- return FunctionConfigUtils.convert(functionConfig, (ClassLoader) null);
- }
}
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java
similarity index 65%
copy from pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java
copy to pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java
index b50e734e0d5..d6aa13dc2bf 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java
@@ -16,87 +16,96 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pulsar.functions.worker.rest.api.v2;
+package org.apache.pulsar.functions.worker.rest.api.v3;
-import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
-import static org.apache.pulsar.functions.utils.FunctionCommon.mergeJson;
import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import com.google.common.collect.Lists;
-import com.google.gson.Gson;
-import com.google.protobuf.InvalidProtocolBufferException;
-import com.google.protobuf.util.JsonFormat;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Paths;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.UUID;
import java.util.function.Consumer;
import javax.ws.rs.core.Response;
-import javax.ws.rs.core.StreamingOutput;
import org.apache.distributedlog.api.namespace.Namespace;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.core.config.Configurator;
import org.apache.pulsar.broker.authentication.AuthenticationParameters;
-import org.apache.pulsar.client.admin.Functions;
-import org.apache.pulsar.client.admin.Namespaces;
-import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.client.admin.Tenants;
import org.apache.pulsar.common.functions.FunctionConfig;
-import org.apache.pulsar.common.policies.data.TenantInfoImpl;
+import org.apache.pulsar.common.functions.UpdateOptionsImpl;
+import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.common.util.RestException;
import org.apache.pulsar.functions.api.Context;
import org.apache.pulsar.functions.api.Function;
-import org.apache.pulsar.functions.instance.InstanceUtils;
import org.apache.pulsar.functions.proto.Function.FunctionDetails;
import org.apache.pulsar.functions.proto.Function.FunctionMetaData;
-import org.apache.pulsar.functions.proto.Function.PackageLocationMetaData;
-import org.apache.pulsar.functions.proto.Function.ProcessingGuarantees;
-import org.apache.pulsar.functions.proto.Function.SinkSpec;
-import org.apache.pulsar.functions.proto.Function.SourceSpec;
import org.apache.pulsar.functions.proto.Function.SubscriptionType;
-import org.apache.pulsar.functions.runtime.RuntimeFactory;
import org.apache.pulsar.functions.source.TopicSchema;
-import org.apache.pulsar.functions.utils.FunctionCommon;
import org.apache.pulsar.functions.utils.FunctionConfigUtils;
-import org.apache.pulsar.functions.worker.FunctionMetaDataManager;
-import org.apache.pulsar.functions.worker.FunctionRuntimeManager;
-import org.apache.pulsar.functions.worker.LeaderService;
-import org.apache.pulsar.functions.worker.PulsarWorkerService;
import org.apache.pulsar.functions.worker.WorkerConfig;
import org.apache.pulsar.functions.worker.WorkerUtils;
-import org.apache.pulsar.functions.worker.rest.api.FunctionsImpl;
-import org.apache.pulsar.functions.worker.rest.api.FunctionsImplV2;
-import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
-import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
-/**
- * Unit test of {@link FunctionsApiV2Resource}.
- */
-public class FunctionApiV2ResourceTest {
+public abstract class AbstractFunctionApiResourceTest extends AbstractFunctionsResourceTest {
+
+ @Test
+ public void testListFunctionsSuccess() {
+ mockInstanceUtils();
+ final List<String> functions = Lists.newArrayList("test-1", "test-2");
+ final List<FunctionMetaData> metaDataList = new LinkedList<>();
+ FunctionMetaData functionMetaData1 = FunctionMetaData.newBuilder().setFunctionDetails(
+ FunctionDetails.newBuilder().setName("test-1").build()
+ ).build();
+ FunctionMetaData functionMetaData2 = FunctionMetaData.newBuilder().setFunctionDetails(
+ FunctionDetails.newBuilder().setName("test-2").build()
+ ).build();
+ metaDataList.add(functionMetaData1);
+ metaDataList.add(functionMetaData2);
+ when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(metaDataList);
+
+ List<String> functionList = listDefaultFunctions();
+ assertEquals(functions, functionList);
+ }
+
+ @Test
+ public void testOnlyGetSources() {
+ List<String> functions = Lists.newArrayList("test-2");
+ List<FunctionMetaData> functionMetaDataList = new LinkedList<>();
+ FunctionMetaData f1 = FunctionMetaData.newBuilder().setFunctionDetails(
+ FunctionDetails.newBuilder()
+ .setName("test-1")
+ .setComponentType(FunctionDetails.ComponentType.SOURCE)
+ .build()).build();
+ functionMetaDataList.add(f1);
+ FunctionMetaData f2 = FunctionMetaData.newBuilder().setFunctionDetails(
+ FunctionDetails.newBuilder()
+ .setName("test-2")
+ .setComponentType(FunctionDetails.ComponentType.FUNCTION)
+ .build()).build();
+ functionMetaDataList.add(f2);
+ FunctionMetaData f3 = FunctionMetaData.newBuilder().setFunctionDetails(
+ FunctionDetails.newBuilder()
+ .setName("test-3")
+ .setComponentType(FunctionDetails.ComponentType.SINK)
+ .build()).build();
+ functionMetaDataList.add(f3);
+ when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(functionMetaDataList);
+
+ List<String> functionList = listDefaultFunctions();
+ assertEquals(functions, functionList);
+ }
private static final class TestFunction implements Function<String, String> {
@@ -106,133 +115,77 @@ public class FunctionApiV2ResourceTest {
}
}
- private static final String tenant = "test-tenant";
- private static final String namespace = "test-namespace";
- private static final String function = "test-function";
- private static final String outputTopic = "test-output-topic";
- private static final String outputSerdeClassName = TopicSchema.DEFAULT_SERDE;
- private static final String className = TestFunction.class.getName();
- private SubscriptionType subscriptionType = SubscriptionType.FAILOVER;
- private static final Map<String, String> topicsToSerDeClassName = new HashMap<>();
- static {
- topicsToSerDeClassName.put("persistent://public/default/test_src", TopicSchema.DEFAULT_SERDE);
- }
- private static final int parallelism = 1;
-
- private PulsarWorkerService mockedWorkerService;
- private PulsarAdmin mockedPulsarAdmin;
- private Tenants mockedTenants;
- private Namespaces mockedNamespaces;
- private Functions mockedFunctions;
- private TenantInfoImpl mockedTenantInfo;
- private List<String> namespaceList = new LinkedList<>();
- private FunctionMetaDataManager mockedManager;
- private FunctionRuntimeManager mockedFunctionRunTimeManager;
- private RuntimeFactory mockedRuntimeFactory;
- private Namespace mockedNamespace;
- private FunctionsImplV2 resource;
- private InputStream mockedInputStream;
- private FormDataContentDisposition mockedFormData;
- private FunctionMetaData mockedFunctionMetadata;
- private LeaderService mockedLeaderService;
- private PulsarFunctionTestTemporaryDirectory tempDirectory;
- private static Map<String, MockedStatic> mockStaticContexts = new HashMap<>();
-
- @BeforeMethod
- public void setup() throws Exception {
- this.mockedManager = mock(FunctionMetaDataManager.class);
- this.mockedFunctionRunTimeManager = mock(FunctionRuntimeManager.class);
- this.mockedTenantInfo = mock(TenantInfoImpl.class);
- this.mockedRuntimeFactory = mock(RuntimeFactory.class);
- this.mockedInputStream = mock(InputStream.class);
- this.mockedNamespace = mock(Namespace.class);
- this.mockedFormData = mock(FormDataContentDisposition.class);
- when(mockedFormData.getFileName()).thenReturn("test");
- this.mockedPulsarAdmin = mock(PulsarAdmin.class);
- this.mockedTenants = mock(Tenants.class);
- this.mockedNamespaces = mock(Namespaces.class);
- this.mockedFunctions = mock(Functions.class);
- this.mockedLeaderService = mock(LeaderService.class);
- this.mockedFunctionMetadata = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build();
- namespaceList.add(tenant + "/" + namespace);
-
- this.mockedWorkerService = mock(PulsarWorkerService.class);
- when(mockedWorkerService.getFunctionMetaDataManager()).thenReturn(mockedManager);
- when(mockedWorkerService.getLeaderService()).thenReturn(mockedLeaderService);
- when(mockedWorkerService.getFunctionRuntimeManager()).thenReturn(mockedFunctionRunTimeManager);
- when(mockedFunctionRunTimeManager.getRuntimeFactory()).thenReturn(mockedRuntimeFactory);
- when(mockedWorkerService.getDlogNamespace()).thenReturn(mockedNamespace);
- when(mockedWorkerService.isInitialized()).thenReturn(true);
- when(mockedWorkerService.getBrokerAdmin()).thenReturn(mockedPulsarAdmin);
- when(mockedWorkerService.getFunctionAdmin()).thenReturn(mockedPulsarAdmin);
- when(mockedPulsarAdmin.tenants()).thenReturn(mockedTenants);
- when(mockedPulsarAdmin.namespaces()).thenReturn(mockedNamespaces);
- when(mockedPulsarAdmin.functions()).thenReturn(mockedFunctions);
- when(mockedTenants.getTenantInfo(any())).thenReturn(mockedTenantInfo);
- when(mockedNamespaces.getNamespaces(any())).thenReturn(namespaceList);
- when(mockedLeaderService.isLeader()).thenReturn(true);
- when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetadata);
+ private static final class WrongFunction implements Consumer<String> {
+ @Override
+ public void accept(String s) {
- // worker config
- WorkerConfig workerConfig = new WorkerConfig()
- .setWorkerId("test")
- .setWorkerPort(8080)
- .setFunctionMetadataTopicName("pulsar/functions")
- .setNumFunctionPackageReplicas(3)
- .setPulsarServiceUrl("pulsar://localhost:6650/");
- tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName());
- tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig);
- when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig);
+ }
+ }
- FunctionsImpl functions = spy(new FunctionsImpl(() -> mockedWorkerService));
+ protected static final String function = "test-function";
+ protected static final String outputTopic = "test-output-topic";
+ protected static final String outputSerdeClassName = TopicSchema.DEFAULT_SERDE;
+ protected static final String className = TestFunction.class.getName();
+ protected SubscriptionType subscriptionType = SubscriptionType.FAILOVER;
+ protected FunctionMetaData mockedFunctionMetadata;
- this.resource = spy(new FunctionsImplV2(functions));
+ @Override
+ protected void doSetup() {
+ this.mockedFunctionMetadata =
+ FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build();
+ when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetadata);
}
- @AfterMethod(alwaysRun = true)
- public void cleanup() {
- if (tempDirectory != null) {
- tempDirectory.delete();
- }
- mockStaticContexts.values().forEach(MockedStatic::close);
- mockStaticContexts.clear();
+ @Override
+ protected FunctionDetails.ComponentType getComponentType() {
+ return FunctionDetails.ComponentType.FUNCTION;
}
- private <T> void mockStatic(Class<T> classStatic, Consumer<MockedStatic<T>> consumer) {
- final MockedStatic<T> mockedStatic =
- mockStaticContexts.computeIfAbsent(classStatic.getName(), name -> Mockito.mockStatic(classStatic));
- consumer.accept(mockedStatic);
- }
- private void mockWorkerUtils() {
- mockStatic(WorkerUtils.class, ctx -> {
- ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod();
- });
- }
+ abstract protected void registerFunction(String tenant, String namespace, String function, InputStream inputStream,
+ FormDataContentDisposition details, String functionPkgUrl, FunctionConfig functionConfig)
+ throws IOException;
+ abstract protected void updateFunction(String tenant,
+ String namespace,
+ String functionName,
+ InputStream uploadedInputStream,
+ FormDataContentDisposition fileDetail,
+ String functionPkgUrl,
+ FunctionConfig functionConfig,
+ AuthenticationParameters authParams,
+ UpdateOptionsImpl updateOptions) throws IOException;
- private void mockWorkerUtils(Consumer<MockedStatic<WorkerUtils>> consumer) {
- mockStatic(WorkerUtils.class, ctx -> {
- ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod();
- if (consumer != null) {
- consumer.accept(ctx);
- }
- });
- }
+ abstract protected File downloadFunction(final String path, final AuthenticationParameters authParams)
+ throws IOException;
- private void mockInstanceUtils() {
- mockStatic(InstanceUtils.class, ctx -> {
- ctx.when(() -> InstanceUtils.calculateSubjectType(any()))
- .thenReturn(FunctionDetails.ComponentType.FUNCTION);
- });
- }
+ abstract protected void testDeregisterFunctionMissingArguments(
+ String tenant,
+ String namespace,
+ String function
+ );
+
+ abstract protected void deregisterDefaultFunction();
+
+ abstract protected void testGetFunctionMissingArguments(
+ String tenant,
+ String namespace,
+ String function
+ ) throws IOException;
+
+ abstract protected void testListFunctionsMissingArguments(
+ String tenant,
+ String namespace
+ );
+
+ abstract protected List<String> listDefaultFunctions();
//
// Register Functions
//
@Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided")
- public void testRegisterFunctionMissingTenant() {
+ public void testRegisterFunctionMissingTenant() throws IOException {
try {
testRegisterFunctionMissingArguments(
null,
@@ -253,7 +206,7 @@ public class FunctionApiV2ResourceTest {
}
@Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided")
- public void testRegisterFunctionMissingNamespace() {
+ public void testRegisterFunctionMissingNamespace() throws IOException {
try {
testRegisterFunctionMissingArguments(
tenant,
@@ -274,7 +227,7 @@ public class FunctionApiV2ResourceTest {
}
@Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided")
- public void testRegisterFunctionMissingFunctionName() {
+ public void testRegisterFunctionMissingFunctionName() throws IOException {
try {
testRegisterFunctionMissingArguments(
tenant,
@@ -294,8 +247,9 @@ public class FunctionApiV2ResourceTest {
}
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package is not provided")
- public void testRegisterFunctionMissingPackage() {
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package is not "
+ + "provided")
+ public void testRegisterFunctionMissingPackage() throws IOException {
try {
testRegisterFunctionMissingArguments(
tenant,
@@ -315,8 +269,9 @@ public class FunctionApiV2ResourceTest {
}
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "No input topic\\(s\\) specified for the function")
- public void testRegisterFunctionMissingInputTopics() throws Exception {
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "No input topic\\(s\\) "
+ + "specified for the function")
+ public void testRegisterFunctionMissingInputTopics() throws IOException {
try {
testRegisterFunctionMissingArguments(
tenant,
@@ -336,8 +291,9 @@ public class FunctionApiV2ResourceTest {
}
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function Package is not provided")
- public void testRegisterFunctionMissingPackageDetails() {
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function Package is not "
+ + "provided")
+ public void testRegisterFunctionMissingPackageDetails() throws IOException {
try {
testRegisterFunctionMissingArguments(
tenant,
@@ -357,10 +313,9 @@ public class FunctionApiV2ResourceTest {
}
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package does not have"
- + " the correct format. Pulsar cannot determine if the package is a NAR package or JAR package. Function "
- + "classname is not provided and attempts to load it as a NAR package produced the following error.*")
- public void testRegisterFunctionMissingClassName() {
+ @Test(expectedExceptions = RestException.class,
+ expectedExceptionsMessageRegExp = "Function class name is not provided.")
+ public void testRegisterFunctionMissingClassName() throws IOException {
try {
testRegisterFunctionMissingArguments(
tenant,
@@ -380,8 +335,9 @@ public class FunctionApiV2ResourceTest {
}
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function class UnknownClass must be in class path")
- public void testRegisterFunctionWrongClassName() {
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function class UnknownClass "
+ + "must be in class path")
+ public void testRegisterFunctionWrongClassName() throws IOException {
try {
testRegisterFunctionMissingArguments(
tenant,
@@ -401,8 +357,9 @@ public class FunctionApiV2ResourceTest {
}
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function parallelism must be a positive number")
- public void testRegisterFunctionWrongParallelism() {
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function parallelism must be a"
+ + " positive number")
+ public void testRegisterFunctionWrongParallelism() throws IOException {
try {
testRegisterFunctionMissingArguments(
tenant,
@@ -423,8 +380,9 @@ public class FunctionApiV2ResourceTest {
}
@Test(expectedExceptions = RestException.class,
- expectedExceptionsMessageRegExp = "Output topic persistent://public/default/test_src is also being used as an input topic \\(topics must be one or the other\\)")
- public void testRegisterFunctionSameInputOutput() {
+ expectedExceptionsMessageRegExp = "Output topic persistent://public/default/test_src is also being used "
+ + "as an input topic \\(topics must be one or the other\\)")
+ public void testRegisterFunctionSameInputOutput() throws IOException {
try {
testRegisterFunctionMissingArguments(
tenant,
@@ -444,9 +402,9 @@ public class FunctionApiV2ResourceTest {
}
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Output topic " + function +
- "-output-topic/test:" + " is invalid")
- public void testRegisterFunctionWrongOutputTopic() {
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Output topic " + function
+ + "-output-topic/test:" + " is invalid")
+ public void testRegisterFunctionWrongOutputTopic() throws IOException {
try {
testRegisterFunctionMissingArguments(
tenant,
@@ -466,8 +424,9 @@ public class FunctionApiV2ResourceTest {
}
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Encountered error .*. when getting Function package from .*")
- public void testRegisterFunctionHttpUrl() {
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Encountered error .*. when "
+ + "getting Function package from .*")
+ public void testRegisterFunctionHttpUrl() throws IOException {
try {
testRegisterFunctionMissingArguments(
tenant,
@@ -487,6 +446,28 @@ public class FunctionApiV2ResourceTest {
}
}
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function class .*. does not "
+ + "implement the correct interface")
+ public void testRegisterFunctionImplementWrongInterface() throws IOException {
+ try {
+ testRegisterFunctionMissingArguments(
+ tenant,
+ namespace,
+ function,
+ mockedInputStream,
+ topicsToSerDeClassName,
+ mockedFormData,
+ outputTopic,
+ outputSerdeClassName,
+ WrongFunction.class.getName(),
+ parallelism,
+ null);
+ } catch (RestException re) {
+ assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
+ throw re;
+ }
+ }
+
private void testRegisterFunctionMissingArguments(
String tenant,
String namespace,
@@ -498,7 +479,7 @@ public class FunctionApiV2ResourceTest {
String outputSerdeClassName,
String className,
Integer parallelism,
- String functionPkgUrl) {
+ String functionPkgUrl) throws IOException {
FunctionConfig functionConfig = new FunctionConfig();
if (tenant != null) {
functionConfig.setTenant(tenant);
@@ -526,49 +507,38 @@ public class FunctionApiV2ResourceTest {
}
functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
- mockWorkerUtils(ctx -> {
- ctx.when(() -> WorkerUtils.uploadFileToBookkeeper(
- anyString(),
- any(File.class),
- any(Namespace.class)))
- .thenCallRealMethod();
- });
+ registerFunction(tenant, namespace, function, inputStream, details, functionPkgUrl, functionConfig);
- try {
- resource.registerFunction(
- tenant,
- namespace,
- function,
- inputStream,
- details,
- functionPkgUrl,
- JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
- AuthenticationParameters.builder().build());
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException(e);
- }
+ }
+
+ @Test(expectedExceptions = Exception.class, expectedExceptionsMessageRegExp = "Function config is not provided")
+ public void testUpdateMissingFunctionConfig() throws IOException {
+ when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
+ updateFunction(
+ tenant,
+ namespace,
+ function,
+ mockedInputStream,
+ mockedFormData,
+ null,
+ null,
+ null, null);
}
- private void registerDefaultFunction() {
+
+ private void registerDefaultFunction() throws IOException {
+ registerDefaultFunctionWithPackageUrl(null);
+ }
+
+ private void registerDefaultFunctionWithPackageUrl(String packageUrl) throws IOException {
FunctionConfig functionConfig = createDefaultFunctionConfig();
- try {
- resource.registerFunction(
- tenant,
- namespace,
- function,
- mockedInputStream,
- mockedFormData,
- null,
- JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
- AuthenticationParameters.builder().build());
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException(e);
- }
+ registerFunction(tenant, namespace, function, mockedInputStream, mockedFormData, packageUrl, functionConfig);
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function already exists")
- public void testRegisterExistedFunction() {
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function already"
+ + " exists")
+ public void testRegisterExistedFunction() throws IOException {
try {
Configurator.setRootLevel(Level.DEBUG);
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
@@ -581,14 +551,16 @@ public class FunctionApiV2ResourceTest {
@Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure")
- public void testRegisterFunctionUploadFailure() throws Exception {
+ public void testRegisterFunctionUploadFailure() throws IOException {
try {
mockWorkerUtils(ctx -> {
- ctx.when(() -> WorkerUtils.uploadFileToBookkeeper(
- anyString(),
- any(File.class),
- any(Namespace.class)))
- .thenThrow(new IOException("upload failure"));
+ ctx.when(() -> {
+ WorkerUtils.uploadFileToBookkeeper(
+ anyString(),
+ any(File.class),
+ any(Namespace.class));
+ }
+ ).thenThrow(new IOException("upload failure"));
});
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
@@ -601,39 +573,38 @@ public class FunctionApiV2ResourceTest {
}
@Test
- public void testRegisterFunctionSuccess() throws Exception {
+ public void testRegisterFunctionSuccess() throws IOException {
try {
- try (MockedStatic mocked = Mockito.mockStatic(WorkerUtils.class)) {
- mocked.when(() -> WorkerUtils.uploadToBookKeeper(
- any(Namespace.class),
- any(InputStream.class),
- anyString())).thenAnswer((i) -> null);
- mocked.when(() -> WorkerUtils.dumpToTmpFile(any())).thenAnswer(i ->
- {
- try {
- File tmpFile = FunctionCommon.createPkgTempFile();
- tmpFile.deleteOnExit();
- Files.copy((InputStream) i.getArguments()[0], tmpFile.toPath(), REPLACE_EXISTING);
- return tmpFile;
- } catch (IOException e) {
- throw new RuntimeException("Cannot create a temporary file", e);
- }
+ mockWorkerUtils();
- }
- );
- WorkerUtils.uploadToBookKeeper(null, null, null);
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
+ when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
- registerDefaultFunction();
- }
+ registerDefaultFunction();
} catch (RestException re) {
assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
throw re;
}
}
+ @Test(timeOut = 20000)
+ public void testRegisterFunctionSuccessWithPackageName() throws IOException {
+ registerDefaultFunctionWithPackageUrl("function://public/default/test@v1");
+ }
+
+ @Test(timeOut = 20000)
+ public void testRegisterFunctionFailedWithWrongPackageName() throws PulsarAdminException, IOException {
+ try {
+ doThrow(new PulsarAdminException("package name is invalid"))
+ .when(mockedPackages).download(anyString(), anyString());
+ registerDefaultFunctionWithPackageUrl("function://");
+ } catch (RestException e) {
+ // expected exception
+ assertEquals(e.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
+ }
+ }
+
@Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace does not exist")
- public void testRegisterFunctionNonExistingNamespace() {
+ public void testRegisterFunctionNonExistingNamespace() throws IOException {
try {
this.namespaceList.clear();
registerDefaultFunction();
@@ -660,8 +631,9 @@ public class FunctionApiV2ResourceTest {
mockWorkerUtils();
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
+
doThrow(new IllegalArgumentException("function failed to register"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean());
+ .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean());
registerDefaultFunction();
} catch (RestException re) {
@@ -670,9 +642,8 @@ public class FunctionApiV2ResourceTest {
}
}
-
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registration interrupted")
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registration "
+ + "interrupted")
public void testRegisterFunctionInterrupted() throws Exception {
try {
mockWorkerUtils();
@@ -680,7 +651,8 @@ public class FunctionApiV2ResourceTest {
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
doThrow(new IllegalStateException("Function registration interrupted"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean());
+ .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean());
+
registerDefaultFunction();
} catch (RestException re) {
assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR);
@@ -932,26 +904,26 @@ public class FunctionApiV2ResourceTest {
if (expectedError != null) {
doThrow(new IllegalArgumentException(expectedError))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean());
+ .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean());
}
- try {
- resource.updateFunction(
- tenant,
- namespace,
- function,
- inputStream,
- details,
- null,
- JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
- AuthenticationParameters.builder().build());
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException(e);
- }
+ updateFunction(
+ tenant,
+ namespace,
+ function,
+ inputStream,
+ details,
+ null,
+ functionConfig,
+ null, null);
}
- private void updateDefaultFunction() {
+ private void updateDefaultFunction() throws IOException {
+ updateDefaultFunctionWithPackageUrl(null);
+ }
+
+ private void updateDefaultFunctionWithPackageUrl(String packageUrl) throws IOException {
FunctionConfig functionConfig = new FunctionConfig();
functionConfig.setTenant(tenant);
functionConfig.setNamespace(namespace);
@@ -963,23 +935,20 @@ public class FunctionApiV2ResourceTest {
functionConfig.setOutput(outputTopic);
functionConfig.setOutputSerdeClassName(outputSerdeClassName);
- try {
- resource.updateFunction(
- tenant,
- namespace,
- function,
- mockedInputStream,
- mockedFormData,
- null,
- JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
- AuthenticationParameters.builder().build());
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException(e);
- }
+ updateFunction(
+ tenant,
+ namespace,
+ function,
+ mockedInputStream,
+ mockedFormData,
+ packageUrl,
+ functionConfig,
+ null, null);
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist")
- public void testUpdateNotExistedFunction() {
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't"
+ + " exist")
+ public void testUpdateNotExistedFunction() throws IOException {
try {
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
updateDefaultFunction();
@@ -993,11 +962,13 @@ public class FunctionApiV2ResourceTest {
public void testUpdateFunctionUploadFailure() throws Exception {
try {
mockWorkerUtils(ctx -> {
- ctx.when(() -> WorkerUtils.uploadFileToBookkeeper(
- anyString(),
- any(File.class),
- any(Namespace.class)))
- .thenThrow(new IOException("upload failure"));
+ ctx.when(() -> {
+ WorkerUtils.uploadFileToBookkeeper(
+ anyString(),
+ any(File.class),
+ any(Namespace.class));
+
+ }).thenThrow(new IOException("upload failure"));
});
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
@@ -1019,13 +990,11 @@ public class FunctionApiV2ResourceTest {
}
@Test
- public void testUpdateFunctionWithUrl() throws Exception {
+ public void testUpdateFunctionWithUrl() throws IOException {
Configurator.setRootLevel(Level.DEBUG);
- URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
- File file = Paths.get(fileUrl.toURI()).toFile();
- String fileLocation = file.getAbsolutePath().replace('\\', '/');
- String filePackageUrl = "file:///" + fileLocation;
+ String fileLocation = FutureUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath();
+ String filePackageUrl = "file://" + fileLocation;
FunctionConfig functionConfig = new FunctionConfig();
functionConfig.setOutput(outputTopic);
@@ -1040,19 +1009,15 @@ public class FunctionApiV2ResourceTest {
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
- try {
- resource.updateFunction(
- tenant,
- namespace,
- function,
- null,
- null,
- filePackageUrl,
- JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
- AuthenticationParameters.builder().build());
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException(e);
- }
+ updateFunction(
+ tenant,
+ namespace,
+ function,
+ null,
+ null,
+ filePackageUrl,
+ functionConfig,
+ null, null);
}
@@ -1064,7 +1029,7 @@ public class FunctionApiV2ResourceTest {
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
doThrow(new IllegalArgumentException("function failed to register"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean());
+ .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean());
updateDefaultFunction();
} catch (RestException re) {
@@ -1073,14 +1038,16 @@ public class FunctionApiV2ResourceTest {
}
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registeration interrupted")
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registeration "
+ + "interrupted")
public void testUpdateFunctionInterrupted() throws Exception {
try {
mockWorkerUtils();
+
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
doThrow(new IllegalStateException("Function registeration interrupted"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean());
+ .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean());
updateDefaultFunction();
} catch (RestException re) {
@@ -1089,6 +1056,26 @@ public class FunctionApiV2ResourceTest {
}
}
+
+ @Test(timeOut = 20000)
+ public void testUpdateFunctionSuccessWithPackageName() throws IOException {
+ when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
+ updateDefaultFunctionWithPackageUrl("function://public/default/test@v1");
+ }
+
+ @Test(timeOut = 20000)
+ public void testUpdateFunctionFailedWithWrongPackageName() throws PulsarAdminException, IOException {
+ when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
+ try {
+ doThrow(new PulsarAdminException("package name is invalid"))
+ .when(mockedPackages).download(anyString(), anyString());
+ registerDefaultFunctionWithPackageUrl("function://");
+ } catch (RestException e) {
+ // expected exception
+ assertEquals(e.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
+ }
+ }
+
//
// deregister function
//
@@ -1136,27 +1123,8 @@ public class FunctionApiV2ResourceTest {
}
}
- private void testDeregisterFunctionMissingArguments(
- String tenant,
- String namespace,
- String function
- ) {
- resource.deregisterFunction(
- tenant,
- namespace,
- function,
- AuthenticationParameters.builder().build());
- }
-
- private void deregisterDefaultFunction() {
- resource.deregisterFunction(
- tenant,
- namespace,
- function,
- AuthenticationParameters.builder().build());
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist")
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't"
+ + " exist")
public void testDeregisterNotExistedFunction() {
try {
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
@@ -1180,7 +1148,7 @@ public class FunctionApiV2ResourceTest {
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
doThrow(new IllegalArgumentException("function failed to deregister"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean());
+ .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean());
deregisterDefaultFunction();
} catch (RestException re) {
@@ -1189,13 +1157,14 @@ public class FunctionApiV2ResourceTest {
}
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function deregisteration interrupted")
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function deregisteration "
+ + "interrupted")
public void testDeregisterFunctionInterrupted() throws Exception {
try {
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
doThrow(new IllegalStateException("Function deregisteration interrupted"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean());
+ .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean());
deregisterDefaultFunction();
} catch (RestException re) {
@@ -1250,76 +1219,6 @@ public class FunctionApiV2ResourceTest {
}
}
- private void testGetFunctionMissingArguments(
- String tenant,
- String namespace,
- String function
- ) throws IOException {
- resource.getFunctionInfo(
- tenant,
- namespace,
- function,
- AuthenticationParameters.builder().build()
- );
-
- }
-
- private FunctionDetails getDefaultFunctionInfo() throws IOException {
- String json = (String) resource.getFunctionInfo(
- tenant,
- namespace,
- function,
- AuthenticationParameters.builder().build()
- ).getEntity();
- FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
- mergeJson(json, functionDetailsBuilder);
- return functionDetailsBuilder.build();
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist")
- public void testGetNotExistedFunction() throws IOException {
- try {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
- getDefaultFunctionInfo();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.NOT_FOUND);
- throw re;
- }
- }
-
- @Test
- public void testGetFunctionSuccess() throws IOException {
- mockInstanceUtils();
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- SinkSpec sinkSpec = SinkSpec.newBuilder()
- .setTopic(outputTopic)
- .setSerDeClassName(outputSerdeClassName).build();
- FunctionDetails functionDetails = FunctionDetails.newBuilder()
- .setClassName(className)
- .setSink(sinkSpec)
- .setName(function)
- .setNamespace(namespace)
- .setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE)
- .setAutoAck(true)
- .setTenant(tenant)
- .setParallelism(parallelism)
- .setSource(SourceSpec.newBuilder().setSubscriptionType(subscriptionType)
- .putAllTopicsToSerDeClassName(topicsToSerDeClassName)).build();
- FunctionMetaData metaData = FunctionMetaData.newBuilder()
- .setCreateTime(System.currentTimeMillis())
- .setFunctionDetails(functionDetails)
- .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("/path/to/package"))
- .setVersion(1234)
- .build();
- when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData);
-
- FunctionDetails actual = getDefaultFunctionInfo();
- assertEquals(
- functionDetails,
- actual);
- }
-
//
// List Functions
//
@@ -1350,88 +1249,61 @@ public class FunctionApiV2ResourceTest {
}
}
- private void testListFunctionsMissingArguments(
- String tenant,
- String namespace
- ) {
- resource.listFunctions(
- tenant,
- namespace,
- AuthenticationParameters.builder().build()
- );
-
+ @Test
+ public void testDownloadFunctionHttpUrl() throws Exception {
+ String jarHttpUrl =
+ "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar";
+ File pkgFile = downloadFunction(jarHttpUrl, null);
+ pkgFile.delete();
}
- private List<String> listDefaultFunctions() {
- return new Gson().fromJson((String) resource.listFunctions(
- tenant,
- namespace,
- AuthenticationParameters.builder().build()
- ).getEntity(), List.class);
+ @Test
+ public void testDownloadFunctionFile() throws Exception {
+ File file = getPulsarApiExamplesNar();
+ File pkgFile = downloadFunction(file.toURI().toString(), null);
+ Assert.assertTrue(pkgFile.exists());
+ Assert.assertEquals(file.length(), pkgFile.length());
+ pkgFile.delete();
}
@Test
- public void testListFunctionsSuccess() {
- mockInstanceUtils();
- final List<String> functions = Lists.newArrayList("test-1", "test-2");
- final List<FunctionMetaData> metaDataList = new LinkedList<>();
- FunctionMetaData functionMetaData1 = FunctionMetaData.newBuilder().setFunctionDetails(
- FunctionDetails.newBuilder().setName("test-1").build()
- ).build();
- FunctionMetaData functionMetaData2 = FunctionMetaData.newBuilder().setFunctionDetails(
- FunctionDetails.newBuilder().setName("test-2").build()
- ).build();
- metaDataList.add(functionMetaData1);
- metaDataList.add(functionMetaData2);
- when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(metaDataList);
+ public void testDownloadFunctionBuiltinConnector() throws Exception {
+ File file = getPulsarApiExamplesNar();
- List<String> functionList = listDefaultFunctions();
- assertEquals(functions, functionList);
- }
+ WorkerConfig config = new WorkerConfig()
+ .setUploadBuiltinSinksSources(false);
+ when(mockedWorkerService.getWorkerConfig()).thenReturn(config);
- @Test
- public void testDownloadFunctionHttpUrl() throws Exception {
- String jarHttpUrl =
- "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar";
- String testDir = FunctionApiV2ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath();
- FunctionsImplV2 function = new FunctionsImplV2(() -> mockedWorkerService);
- StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction(jarHttpUrl,
- AuthenticationParameters.builder().build()).getEntity();
- File pkgFile = new File(testDir, UUID.randomUUID().toString());
- OutputStream output = new FileOutputStream(pkgFile);
- streamOutput.write(output);
+ registerBuiltinConnector("cassandra", file);
+
+ File pkgFile = downloadFunction("builtin://cassandra", null);
Assert.assertTrue(pkgFile.exists());
- if (pkgFile.exists()) {
- pkgFile.delete();
- }
+ Assert.assertEquals(file.length(), pkgFile.length());
+ pkgFile.delete();
}
@Test
- public void testDownloadFunctionFile() throws Exception {
- URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
- File file = Paths.get(fileUrl.toURI()).toFile();
- String fileLocation = file.getAbsolutePath().replace('\\', '/');
- String testDir = FunctionApiV2ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath();
- FunctionsImplV2 function = new FunctionsImplV2(() -> mockedWorkerService);
- StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction("file:///" + fileLocation,
- AuthenticationParameters.builder().build()).getEntity();
- File pkgFile = new File(testDir, UUID.randomUUID().toString());
- OutputStream output = new FileOutputStream(pkgFile);
- streamOutput.write(output);
+ public void testDownloadFunctionBuiltinFunction() throws Exception {
+ File file = getPulsarApiExamplesNar();
+
+ WorkerConfig config = new WorkerConfig()
+ .setUploadBuiltinSinksSources(false);
+ when(mockedWorkerService.getWorkerConfig()).thenReturn(config);
+
+ registerBuiltinFunction("exclamation", file);
+
+ File pkgFile = downloadFunction("builtin://exclamation", null);
Assert.assertTrue(pkgFile.exists());
- if (pkgFile.exists()) {
- pkgFile.delete();
- }
+ Assert.assertEquals(file.length(), pkgFile.length());
+ pkgFile.delete();
}
@Test
public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception {
Configurator.setRootLevel(Level.DEBUG);
- URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
- File file = Paths.get(fileUrl.toURI()).toFile();
- String fileLocation = file.getAbsolutePath().replace('\\', '/');
- String filePackageUrl = "file:///" + fileLocation;
+ File file = getPulsarApiExamplesNar();
+ String filePackageUrl = file.toURI().toString();
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
FunctionConfig functionConfig = new FunctionConfig();
@@ -1444,13 +1316,7 @@ public class FunctionApiV2ResourceTest {
functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
functionConfig.setOutput(outputTopic);
functionConfig.setOutputSerdeClassName(outputSerdeClassName);
- try {
- resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl,
- JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
- AuthenticationParameters.builder().build());
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException(e);
- }
+ registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig);
}
@@ -1462,10 +1328,8 @@ public class FunctionApiV2ResourceTest {
String actualName = "DIFFERENT_NAME";
this.namespaceList.add(actualTenant + "/" + actualNamespace);
- URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
- File file = Paths.get(fileUrl.toURI()).toFile();
- String fileLocation = file.getAbsolutePath().replace('\\', '/');
- String filePackageUrl = "file:///" + fileLocation;
+ File file = getPulsarApiExamplesNar();
+ String filePackageUrl = file.toURI().toString();
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
when(mockedManager.containsFunction(eq(actualTenant), eq(actualNamespace), eq(actualName))).thenReturn(false);
@@ -1479,13 +1343,7 @@ public class FunctionApiV2ResourceTest {
functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
functionConfig.setOutput(outputTopic);
functionConfig.setOutputSerdeClassName(outputSerdeClassName);
- try {
- resource.registerFunction(actualTenant, actualNamespace, actualName, null, null, filePackageUrl,
- JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)),
- AuthenticationParameters.builder().build());
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException(e);
- }
+ registerFunction(actualTenant, actualNamespace, actualName, null, null, filePackageUrl, functionConfig);
}
public static FunctionConfig createDefaultFunctionConfig() {
@@ -1504,6 +1362,6 @@ public class FunctionApiV2ResourceTest {
public static FunctionDetails createDefaultFunctionDetails() {
FunctionConfig functionConfig = createDefaultFunctionConfig();
- return FunctionConfigUtils.convert(functionConfig, (ClassLoader) null);
+ return FunctionConfigUtils.convert(functionConfig);
}
}
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionsResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionsResourceTest.java
new file mode 100644
index 00000000000..9da30128f1c
--- /dev/null
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionsResourceTest.java
@@ -0,0 +1,323 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.functions.worker.rest.api.v3;
+
+import static org.apache.pulsar.functions.source.TopicSchema.DEFAULT_SERDE;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.withSettings;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Consumer;
+import org.apache.distributedlog.api.namespace.Namespace;
+import org.apache.pulsar.client.admin.Functions;
+import org.apache.pulsar.client.admin.Namespaces;
+import org.apache.pulsar.client.admin.Packages;
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.admin.Tenants;
+import org.apache.pulsar.common.functions.FunctionDefinition;
+import org.apache.pulsar.common.io.ConnectorDefinition;
+import org.apache.pulsar.common.nar.NarClassLoader;
+import org.apache.pulsar.common.policies.data.TenantInfoImpl;
+import org.apache.pulsar.functions.instance.InstanceUtils;
+import org.apache.pulsar.functions.proto.Function;
+import org.apache.pulsar.functions.runtime.RuntimeFactory;
+import org.apache.pulsar.functions.source.TopicSchema;
+import org.apache.pulsar.functions.utils.LoadedFunctionPackage;
+import org.apache.pulsar.functions.utils.ValidatableFunctionPackage;
+import org.apache.pulsar.functions.utils.functions.FunctionArchive;
+import org.apache.pulsar.functions.utils.functions.FunctionUtils;
+import org.apache.pulsar.functions.utils.io.Connector;
+import org.apache.pulsar.functions.utils.io.ConnectorUtils;
+import org.apache.pulsar.functions.worker.ConnectorsManager;
+import org.apache.pulsar.functions.worker.FunctionMetaDataManager;
+import org.apache.pulsar.functions.worker.FunctionRuntimeManager;
+import org.apache.pulsar.functions.worker.FunctionsManager;
+import org.apache.pulsar.functions.worker.LeaderService;
+import org.apache.pulsar.functions.worker.PulsarWorkerService;
+import org.apache.pulsar.functions.worker.WorkerConfig;
+import org.apache.pulsar.functions.worker.WorkerUtils;
+import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory;
+import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
+import org.mockito.Answers;
+import org.mockito.MockSettings;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+
+public abstract class AbstractFunctionsResourceTest {
+
+ protected static final String tenant = "test-tenant";
+ protected static final String namespace = "test-namespace";
+ protected static final Map<String, String> topicsToSerDeClassName = new HashMap<>();
+ protected static final String subscriptionName = "test-subscription";
+ protected static final String CASSANDRA_STRING_SINK = "org.apache.pulsar.io.cassandra.CassandraStringSink";
+ protected static final int parallelism = 1;
+ private static final String SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH = "pulsar-io-cassandra.nar.path";
+ private static final String SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH = "pulsar-io-twitter.nar.path";
+ private static final String SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH = "pulsar-io-invalid.nar.path";
+ private static final String SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH =
+ "pulsar-functions-api-examples.nar.path";
+ protected static Map<String, MockedStatic> mockStaticContexts = new HashMap<>();
+
+ static {
+ topicsToSerDeClassName.put("test_src", DEFAULT_SERDE);
+ topicsToSerDeClassName.put("persistent://public/default/test_src", TopicSchema.DEFAULT_SERDE);
+ }
+
+ protected PulsarWorkerService mockedWorkerService;
+ protected PulsarAdmin mockedPulsarAdmin;
+ protected Tenants mockedTenants;
+ protected Namespaces mockedNamespaces;
+ protected Functions mockedFunctions;
+ protected TenantInfoImpl mockedTenantInfo;
+ protected List<String> namespaceList = new LinkedList<>();
+ protected FunctionMetaDataManager mockedManager;
+ protected FunctionRuntimeManager mockedFunctionRunTimeManager;
+ protected RuntimeFactory mockedRuntimeFactory;
+ protected Namespace mockedNamespace;
+ protected InputStream mockedInputStream;
+ protected FormDataContentDisposition mockedFormData;
+ protected Function.FunctionMetaData mockedFunctionMetaData;
+ protected LeaderService mockedLeaderService;
+ protected Packages mockedPackages;
+ protected PulsarFunctionTestTemporaryDirectory tempDirectory;
+ protected ConnectorsManager connectorsManager = new ConnectorsManager();
+ protected FunctionsManager functionsManager = new FunctionsManager();
+
+ public static File getPulsarIOCassandraNar() {
+ return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH)
+ , "pulsar-io-cassandra.nar file location must be specified with "
+ + SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH + " system property"));
+ }
+
+ public static File getPulsarIOTwitterNar() {
+ return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH)
+ , "pulsar-io-twitter.nar file location must be specified with "
+ + SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH + " system property"));
+ }
+
+ public static File getPulsarIOInvalidNar() {
+ return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH)
+ , "invalid nar file location must be specified with "
+ + SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH + " system property"));
+ }
+
+ public static File getPulsarApiExamplesNar() {
+ return new File(Objects.requireNonNull(
+ System.getProperty(SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH)
+ , "pulsar-functions-api-examples.nar file location must be specified with "
+ + SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH + " system property"));
+ }
+
+ @BeforeMethod
+ public final void setup() throws Exception {
+ this.mockedManager = mock(FunctionMetaDataManager.class);
+ this.mockedFunctionRunTimeManager = mock(FunctionRuntimeManager.class);
+ this.mockedRuntimeFactory = mock(RuntimeFactory.class);
+ this.mockedInputStream = mock(InputStream.class);
+ this.mockedNamespace = mock(Namespace.class);
+ this.mockedFormData = mock(FormDataContentDisposition.class);
+ when(mockedFormData.getFileName()).thenReturn("test");
+ this.mockedTenantInfo = mock(TenantInfoImpl.class);
+ this.mockedPulsarAdmin = mock(PulsarAdmin.class);
+ this.mockedTenants = mock(Tenants.class);
+ this.mockedNamespaces = mock(Namespaces.class);
+ this.mockedFunctions = mock(Functions.class);
+ this.mockedLeaderService = mock(LeaderService.class);
+ this.mockedPackages = mock(Packages.class);
+ namespaceList.add(tenant + "/" + namespace);
+
+ this.mockedWorkerService = mock(PulsarWorkerService.class);
+ when(mockedWorkerService.getFunctionMetaDataManager()).thenReturn(mockedManager);
+ when(mockedWorkerService.getLeaderService()).thenReturn(mockedLeaderService);
+ when(mockedWorkerService.getFunctionRuntimeManager()).thenReturn(mockedFunctionRunTimeManager);
+ when(mockedFunctionRunTimeManager.getRuntimeFactory()).thenReturn(mockedRuntimeFactory);
+ when(mockedWorkerService.getDlogNamespace()).thenReturn(mockedNamespace);
+ when(mockedWorkerService.isInitialized()).thenReturn(true);
+ when(mockedWorkerService.getBrokerAdmin()).thenReturn(mockedPulsarAdmin);
+ when(mockedWorkerService.getFunctionAdmin()).thenReturn(mockedPulsarAdmin);
+ when(mockedPulsarAdmin.tenants()).thenReturn(mockedTenants);
+ when(mockedPulsarAdmin.namespaces()).thenReturn(mockedNamespaces);
+ when(mockedPulsarAdmin.functions()).thenReturn(mockedFunctions);
+ when(mockedPulsarAdmin.packages()).thenReturn(mockedPackages);
+ when(mockedTenants.getTenantInfo(any())).thenReturn(mockedTenantInfo);
+ when(mockedNamespaces.getNamespaces(any())).thenReturn(namespaceList);
+ when(mockedLeaderService.isLeader()).thenReturn(true);
+ doAnswer(invocationOnMock -> {
+ Files.copy(getDefaultNarFile().toPath(), Paths.get(invocationOnMock.getArgument(1, String.class)),
+ StandardCopyOption.REPLACE_EXISTING);
+ return null;
+ }).when(mockedPackages).download(any(), any());
+
+ // worker config
+ WorkerConfig workerConfig = new WorkerConfig()
+ .setWorkerId("test")
+ .setWorkerPort(8080)
+ .setFunctionMetadataTopicName("pulsar/functions")
+ .setNumFunctionPackageReplicas(3)
+ .setPulsarServiceUrl("pulsar://localhost:6650/");
+ tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName());
+ tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig);
+ when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig);
+ when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager);
+ when(mockedWorkerService.getConnectorsManager()).thenReturn(connectorsManager);
+
+ doSetup();
+ }
+
+ protected File getDefaultNarFile() {
+ return getPulsarIOTwitterNar();
+ }
+
+ protected void doSetup() throws Exception {
+
+ }
+
+ @AfterMethod(alwaysRun = true)
+ public void cleanup() {
+ if (tempDirectory != null) {
+ tempDirectory.delete();
+ }
+ mockStaticContexts.values().forEach(MockedStatic::close);
+ mockStaticContexts.clear();
+ }
+
+ protected <T> void mockStatic(Class<T> classStatic, Consumer<MockedStatic<T>> consumer) {
+ mockStatic(classStatic, withSettings().defaultAnswer(Mockito.CALLS_REAL_METHODS), consumer);
+ }
+
+ private <T> void mockStatic(Class<T> classStatic, MockSettings mockSettings, Consumer<MockedStatic<T>> consumer) {
+ final MockedStatic<T> mockedStatic = mockStaticContexts.computeIfAbsent(classStatic.getName(),
+ name -> Mockito.mockStatic(classStatic, mockSettings));
+ consumer.accept(mockedStatic);
+ }
+
+ protected void mockWorkerUtils() {
+ mockWorkerUtils(null);
+ }
+
+ protected void mockWorkerUtils(Consumer<MockedStatic<WorkerUtils>> consumer) {
+ mockStatic(WorkerUtils.class, withSettings(), ctx -> {
+ // make dumpToTmpFile return the nar file copy
+ ctx.when(() -> WorkerUtils.dumpToTmpFile(mockedInputStream))
+ .thenAnswer(invocation -> {
+ Path tempFile = Files.createTempFile("test", ".nar");
+ Files.copy(getPulsarApiExamplesNar().toPath(), tempFile,
+ StandardCopyOption.REPLACE_EXISTING);
+ return tempFile.toFile();
+ });
+ ctx.when(() -> WorkerUtils.dumpToTmpFile(any()))
+ .thenAnswer(Answers.CALLS_REAL_METHODS);
+ if (consumer != null) {
+ consumer.accept(ctx);
+ }
+ });
+ }
+
+ protected void mockInstanceUtils() {
+ mockStatic(InstanceUtils.class, ctx -> {
+ ctx.when(() -> InstanceUtils.calculateSubjectType(any()))
+ .thenReturn(getComponentType());
+ });
+ }
+
+ protected abstract Function.FunctionDetails.ComponentType getComponentType();
+
+ public static class LoadedConnector extends Connector {
+ public LoadedConnector(ConnectorDefinition connectorDefinition) {
+ super(null, connectorDefinition, null, true);
+ }
+
+ @Override
+ public ValidatableFunctionPackage getConnectorFunctionPackage() {
+ return new LoadedFunctionPackage(getClass().getClassLoader(), ConnectorDefinition.class,
+ getConnectorDefinition());
+ }
+ }
+
+
+ protected void registerBuiltinConnector(String connectorType, String className) {
+ ConnectorDefinition connectorDefinition = null;
+ if (className != null) {
+ connectorDefinition = new ConnectorDefinition();
+ // set source and sink class to the same to simplify the test
+ connectorDefinition.setSinkClass(className);
+ connectorDefinition.setSourceClass(className);
+ }
+ connectorsManager.addConnector(connectorType, new LoadedConnector(connectorDefinition));
+ }
+
+ protected void registerBuiltinConnector(String connectorType, File packageFile) {
+ ConnectorDefinition cntDef;
+ try {
+ cntDef = ConnectorUtils.getConnectorDefinition(packageFile);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ connectorsManager.addConnector(connectorType,
+ new Connector(packageFile.toPath(), cntDef, NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, true));
+ }
+
+ public static class LoadedFunctionArchive extends FunctionArchive {
+ public LoadedFunctionArchive(FunctionDefinition functionDefinition) {
+ super(null, functionDefinition, null, true);
+ }
+
+ @Override
+ public ValidatableFunctionPackage getFunctionPackage() {
+ return new LoadedFunctionPackage(getClass().getClassLoader(), FunctionDefinition.class,
+ getFunctionDefinition());
+ }
+ }
+
+ protected void registerBuiltinFunction(String functionType, String className) {
+ FunctionDefinition functionDefinition = null;
+ if (className != null) {
+ functionDefinition = new FunctionDefinition();
+ functionDefinition.setFunctionClass(className);
+ }
+ functionsManager.addFunction(functionType, new LoadedFunctionArchive(functionDefinition));
+ }
+
+ protected void registerBuiltinFunction(String functionType, File packageFile) {
+ FunctionDefinition cntDef;
+ try {
+ cntDef = FunctionUtils.getFunctionDefinition(packageFile);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ functionsManager.addFunction(functionType,
+ new FunctionArchive(packageFile.toPath(), cntDef, NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, true));
+ }
+}
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java
index e449e8669cf..ae3c0255ea6 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java
@@ -21,1693 +21,188 @@ package org.apache.pulsar.functions.worker.rest.api.v3;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
-import com.google.common.collect.Lists;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.net.URL;
-import java.nio.file.Paths;
-import java.util.HashMap;
-import java.util.LinkedList;
import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.UUID;
-import java.util.function.Consumer;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
import org.apache.distributedlog.api.namespace.Namespace;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.core.config.Configurator;
import org.apache.pulsar.broker.authentication.AuthenticationParameters;
-import org.apache.pulsar.client.admin.Functions;
-import org.apache.pulsar.client.admin.Namespaces;
-import org.apache.pulsar.client.admin.Packages;
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.client.admin.Tenants;
import org.apache.pulsar.common.functions.FunctionConfig;
-import org.apache.pulsar.common.nar.NarClassLoader;
-import org.apache.pulsar.common.policies.data.TenantInfoImpl;
-import org.apache.pulsar.common.util.FutureUtil;
+import org.apache.pulsar.common.functions.UpdateOptionsImpl;
import org.apache.pulsar.common.util.RestException;
-import org.apache.pulsar.functions.api.Context;
-import org.apache.pulsar.functions.api.Function;
-import org.apache.pulsar.functions.instance.InstanceUtils;
-import org.apache.pulsar.functions.proto.Function.FunctionDetails;
-import org.apache.pulsar.functions.proto.Function.FunctionMetaData;
-import org.apache.pulsar.functions.proto.Function.PackageLocationMetaData;
-import org.apache.pulsar.functions.proto.Function.ProcessingGuarantees;
-import org.apache.pulsar.functions.proto.Function.SinkSpec;
-import org.apache.pulsar.functions.proto.Function.SourceSpec;
-import org.apache.pulsar.functions.proto.Function.SubscriptionType;
-import org.apache.pulsar.functions.runtime.RuntimeFactory;
-import org.apache.pulsar.functions.source.TopicSchema;
-import org.apache.pulsar.functions.utils.FunctionCommon;
+import org.apache.pulsar.functions.proto.Function;
import org.apache.pulsar.functions.utils.FunctionConfigUtils;
-import org.apache.pulsar.functions.utils.functions.FunctionArchive;
-import org.apache.pulsar.functions.utils.io.Connector;
-import org.apache.pulsar.functions.worker.ConnectorsManager;
-import org.apache.pulsar.functions.worker.FunctionMetaDataManager;
-import org.apache.pulsar.functions.worker.FunctionRuntimeManager;
-import org.apache.pulsar.functions.worker.FunctionsManager;
-import org.apache.pulsar.functions.worker.LeaderService;
-import org.apache.pulsar.functions.worker.PulsarWorkerService;
import org.apache.pulsar.functions.worker.WorkerConfig;
import org.apache.pulsar.functions.worker.WorkerUtils;
import org.apache.pulsar.functions.worker.rest.api.FunctionsImpl;
-import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory;
-import org.apache.pulsar.functions.worker.rest.api.v2.FunctionsApiV2Resource;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
-import org.mockito.MockedStatic;
-import org.mockito.Mockito;
import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
-/**
- * Unit test of {@link FunctionsApiV2Resource}.
- */
-public class FunctionApiV3ResourceTest {
-
- private static final class TestFunction implements Function<String, String> {
-
- @Override
- public String process(String input, Context context) {
- return input;
- }
- }
-
- private static final class WrongFunction implements Consumer<String> {
- @Override
- public void accept(String s) {
-
- }
- }
-
- private static final String tenant = "test-tenant";
- private static final String namespace = "test-namespace";
- private static final String function = "test-function";
- private static final String outputTopic = "test-output-topic";
- private static final String outputSerdeClassName = TopicSchema.DEFAULT_SERDE;
- private static final String className = TestFunction.class.getName();
- private SubscriptionType subscriptionType = SubscriptionType.FAILOVER;
- private static final Map<String, String> topicsToSerDeClassName = new HashMap<>();
- static {
- topicsToSerDeClassName.put("persistent://public/default/test_src", TopicSchema.DEFAULT_SERDE);
- }
- private static final int parallelism = 1;
-
- private PulsarWorkerService mockedWorkerService;
- private PulsarAdmin mockedPulsarAdmin;
- private Tenants mockedTenants;
- private Namespaces mockedNamespaces;
- private Functions mockedFunctions;
- private TenantInfoImpl mockedTenantInfo;
- private List<String> namespaceList = new LinkedList<>();
- private FunctionMetaDataManager mockedManager;
- private FunctionRuntimeManager mockedFunctionRunTimeManager;
- private RuntimeFactory mockedRuntimeFactory;
- private Namespace mockedNamespace;
- private FunctionsImpl resource;
- private InputStream mockedInputStream;
- private FormDataContentDisposition mockedFormData;
- private FunctionMetaData mockedFunctionMetadata;
- private LeaderService mockedLeaderService;
- private Packages mockedPackages;
- private PulsarFunctionTestTemporaryDirectory tempDirectory;
- private static Map<String, MockedStatic> mockStaticContexts = new HashMap<>();
-
- private static final String SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH =
- "pulsar-functions-api-examples.nar.path";
-
- public static File getPulsarApiExamplesNar() {
- return new File(Objects.requireNonNull(
- System.getProperty(SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH)
- , "pulsar-functions-api-examples.nar file location must be specified with "
- + SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH + " system property"));
- }
-
- @BeforeMethod
- public void setup() throws Exception {
- this.mockedManager = mock(FunctionMetaDataManager.class);
- this.mockedFunctionRunTimeManager = mock(FunctionRuntimeManager.class);
- this.mockedTenantInfo = mock(TenantInfoImpl.class);
- this.mockedRuntimeFactory = mock(RuntimeFactory.class);
- this.mockedInputStream = mock(InputStream.class);
- this.mockedNamespace = mock(Namespace.class);
- this.mockedFormData = mock(FormDataContentDisposition.class);
- when(mockedFormData.getFileName()).thenReturn("test");
- this.mockedPulsarAdmin = mock(PulsarAdmin.class);
- this.mockedTenants = mock(Tenants.class);
- this.mockedNamespaces = mock(Namespaces.class);
- this.mockedFunctions = mock(Functions.class);
- this.mockedPackages = mock(Packages.class);
- this.mockedLeaderService = mock(LeaderService.class);
- this.mockedFunctionMetadata = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build();
- namespaceList.add(tenant + "/" + namespace);
-
- this.mockedWorkerService = mock(PulsarWorkerService.class);
- when(mockedWorkerService.getFunctionMetaDataManager()).thenReturn(mockedManager);
- when(mockedWorkerService.getFunctionRuntimeManager()).thenReturn(mockedFunctionRunTimeManager);
- when(mockedWorkerService.getLeaderService()).thenReturn(mockedLeaderService);
- when(mockedFunctionRunTimeManager.getRuntimeFactory()).thenReturn(mockedRuntimeFactory);
- when(mockedWorkerService.getDlogNamespace()).thenReturn(mockedNamespace);
- when(mockedWorkerService.isInitialized()).thenReturn(true);
- when(mockedWorkerService.getBrokerAdmin()).thenReturn(mockedPulsarAdmin);
- when(mockedWorkerService.getFunctionAdmin()).thenReturn(mockedPulsarAdmin);
- when(mockedPulsarAdmin.tenants()).thenReturn(mockedTenants);
- when(mockedPulsarAdmin.namespaces()).thenReturn(mockedNamespaces);
- when(mockedPulsarAdmin.functions()).thenReturn(mockedFunctions);
- when(mockedPulsarAdmin.packages()).thenReturn(mockedPackages);
- when(mockedTenants.getTenantInfo(any())).thenReturn(mockedTenantInfo);
- when(mockedNamespaces.getNamespaces(any())).thenReturn(namespaceList);
- when(mockedLeaderService.isLeader()).thenReturn(true);
- when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetadata);
- doNothing().when(mockedPackages).download(anyString(), anyString());
-
- // worker config
- WorkerConfig workerConfig = new WorkerConfig()
- .setWorkerId("test")
- .setWorkerPort(8080)
- .setFunctionMetadataTopicName("pulsar/functions")
- .setNumFunctionPackageReplicas(3)
- .setPulsarServiceUrl("pulsar://localhost:6650/");
- tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName());
- tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig);
- when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig);
-
- this.resource = spy(new FunctionsImpl(() -> mockedWorkerService));
- }
-
- @AfterMethod(alwaysRun = true)
- public void cleanup() {
- if (tempDirectory != null) {
- tempDirectory.delete();
- }
- mockStaticContexts.values().forEach(MockedStatic::close);
- mockStaticContexts.clear();
- }
-
- private <T> void mockStatic(Class<T> classStatic, Consumer<MockedStatic<T>> consumer) {
- final MockedStatic<T> mockedStatic = mockStaticContexts.computeIfAbsent(classStatic.getName(), name -> Mockito.mockStatic(classStatic));
- consumer.accept(mockedStatic);
- }
-
- private void mockWorkerUtils() {
- mockWorkerUtils(null);
- }
-
- private void mockWorkerUtils(Consumer<MockedStatic<WorkerUtils>> consumer) {
- mockStatic(WorkerUtils.class, ctx -> {
- ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod();
- if (consumer != null) {
- consumer.accept(ctx);
- }
- });
- }
-
- private void mockInstanceUtils() {
- mockStatic(InstanceUtils.class, ctx -> {
- ctx.when(() -> InstanceUtils.calculateSubjectType(any()))
- .thenReturn(FunctionDetails.ComponentType.FUNCTION);
- });
- }
-
- //
- // Register Functions
- //
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided")
- public void testRegisterFunctionMissingTenant() {
- try {
- testRegisterFunctionMissingArguments(
- null,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided")
- public void testRegisterFunctionMissingNamespace() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- null,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- null);
- } catch (RestException re){
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided")
- public void testRegisterFunctionMissingFunctionName() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- null,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- null);
- } catch (RestException re){
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package is not provided")
- public void testRegisterFunctionMissingPackage() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- null,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "No input topic\\(s\\) specified for the function")
- public void testRegisterFunctionMissingInputTopics() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- mockedInputStream,
- null,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function Package is not provided")
- public void testRegisterFunctionMissingPackageDetails() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- null,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- null);
- } catch (RestException re){
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package does not have"
- + " the correct format. Pulsar cannot determine if the package is a NAR package or JAR package. Function "
- + "classname is not provided and attempts to load it as a NAR package produced the following error.*")
- public void testRegisterFunctionMissingClassName() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- null,
- parallelism,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function class UnknownClass must be in class path")
- public void testRegisterFunctionWrongClassName() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- "UnknownClass",
- parallelism,
- null);
- } catch (RestException re){
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function parallelism must be a positive number")
- public void testRegisterFunctionWrongParallelism() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- -2,
- null);
- } catch (RestException re){
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class,
- expectedExceptionsMessageRegExp = "Output topic persistent://public/default/test_src is also being used as an input topic \\(topics must be one or the other\\)")
- public void testRegisterFunctionSameInputOutput() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- topicsToSerDeClassName.keySet().iterator().next(),
- outputSerdeClassName,
- className,
- parallelism,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Output topic " + function + "-output-topic/test:" + " is invalid")
- public void testRegisterFunctionWrongOutputTopic() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- function + "-output-topic/test:",
- outputSerdeClassName,
- className,
- parallelism,
- null);
- } catch (RestException re){
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Encountered error .*. when getting Function package from .*")
- public void testRegisterFunctionHttpUrl() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- null,
- topicsToSerDeClassName,
- null,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- "http://localhost:1234/test");
- } catch (RestException re){
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function class .*. does not implement the correct interface")
- public void testRegisterFunctionImplementWrongInterface() {
- try {
- testRegisterFunctionMissingArguments(
- tenant,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- WrongFunction.class.getName(),
- parallelism,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- private void testRegisterFunctionMissingArguments(
- String tenant,
- String namespace,
- String function,
- InputStream inputStream,
- Map<String, String> topicsToSerDeClassName,
- FormDataContentDisposition details,
- String outputTopic,
- String outputSerdeClassName,
- String className,
- Integer parallelism,
- String functionPkgUrl) {
- FunctionConfig functionConfig = new FunctionConfig();
- if (tenant != null) {
- functionConfig.setTenant(tenant);
- }
- if (namespace != null) {
- functionConfig.setNamespace(namespace);
- }
- if (function != null) {
- functionConfig.setName(function);
- }
- if (topicsToSerDeClassName != null) {
- functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
- }
- if (outputTopic != null) {
- functionConfig.setOutput(outputTopic);
- }
- if (outputSerdeClassName != null) {
- functionConfig.setOutputSerdeClassName(outputSerdeClassName);
- }
- if (className != null) {
- functionConfig.setClassName(className);
- }
- if (parallelism != null) {
- functionConfig.setParallelism(parallelism);
- }
- functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
-
- resource.registerFunction(
- tenant,
- namespace,
- function,
- inputStream,
- details,
- functionPkgUrl,
- functionConfig,
- null);
-
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function config is not provided")
- public void testMissingFunctionConfig() {
- resource.registerFunction(
- tenant,
- namespace,
- function,
- mockedInputStream,
- mockedFormData,
- null,
- null,
- null);
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function config is not provided")
- public void testUpdateMissingFunctionConfig() {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- resource.updateFunction(
- tenant,
- namespace,
- function,
- mockedInputStream,
- mockedFormData,
- null,
- null,
- null, null);
- }
-
- private void registerDefaultFunction() {
- registerDefaultFunctionWithPackageUrl(null);
- }
-
- private void registerDefaultFunctionWithPackageUrl(String packageUrl) {
- FunctionConfig functionConfig = createDefaultFunctionConfig();
- resource.registerFunction(
- tenant,
- namespace,
- function,
- mockedInputStream,
- mockedFormData,
- packageUrl,
- functionConfig,
- null);
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function already exists")
- public void testRegisterExistedFunction() {
- try {
- Configurator.setRootLevel(Level.DEBUG);
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
- registerDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure")
- public void testRegisterFunctionUploadFailure() throws Exception {
- try {
- mockWorkerUtils(ctx -> {
- ctx.when(() -> {
- WorkerUtils.uploadFileToBookkeeper(
- anyString(),
- any(File.class),
- any(Namespace.class));
- }
- ).thenThrow(new IOException("upload failure"));
- ;
- });
-
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
-
- registerDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR);
- throw re;
- }
- }
-
- @Test
- public void testRegisterFunctionSuccess() throws Exception {
- try {
- mockWorkerUtils();
-
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
-
- registerDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(timeOut = 20000)
- public void testRegisterFunctionSuccessWithPackageName() {
- registerDefaultFunctionWithPackageUrl("function://public/default/test@v1");
- }
-
- @Test(timeOut = 20000)
- public void testRegisterFunctionFailedWithWrongPackageName() throws PulsarAdminException {
- try {
- doThrow(new PulsarAdminException("package name is invalid"))
- .when(mockedPackages).download(anyString(), anyString());
- registerDefaultFunctionWithPackageUrl("function://");
- } catch (RestException e) {
- // expected exception
- assertEquals(e.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace does not exist")
- public void testRegisterFunctionNonExistingNamespace() {
- try {
- this.namespaceList.clear();
- registerDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant does not exist")
- public void testRegisterFunctionNonexistantTenant() throws Exception {
- try {
- when(mockedTenants.getTenantInfo(any())).thenThrow(PulsarAdminException.NotFoundException.class);
- registerDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register")
- public void testRegisterFunctionFailure() throws Exception {
- try {
- mockWorkerUtils();
-
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
-
- doThrow(new IllegalArgumentException("function failed to register"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean());
-
- registerDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registration interrupted")
- public void testRegisterFunctionInterrupted() throws Exception {
- try {
- mockWorkerUtils();
-
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
-
- doThrow(new IllegalStateException("Function registration interrupted"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean());
-
- registerDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR);
- throw re;
- }
- }
-
- /*
- Externally managed runtime,
- uploadBuiltinSinksSources == false
- Make sure uploadFileToBookkeeper is not called
- */
- @Test
- public void testRegisterFunctionSuccessK8sNoUpload() throws Exception {
- mockedWorkerService.getWorkerConfig().setUploadBuiltinSinksSources(false);
-
- mockStatic(WorkerUtils.class, ctx -> {
- ctx.when(() -> WorkerUtils.uploadFileToBookkeeper(
- anyString(),
- any(File.class),
- any(Namespace.class)))
- .thenThrow(new RuntimeException("uploadFileToBookkeeper triggered"));
-
- });
-
- NarClassLoader mockedClassLoader = mock(NarClassLoader.class);
- mockStatic(FunctionCommon.class, ctx -> {
- ctx.when(() -> FunctionCommon.getFunctionTypes(any(FunctionConfig.class), any(Class.class))).thenReturn(new Class[]{String.class, String.class});
- ctx.when(() -> FunctionCommon.convertRuntime(any(FunctionConfig.Runtime.class))).thenCallRealMethod();
- ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true);
-
- });
-
- doReturn(Function.class).when(mockedClassLoader).loadClass(anyString());
-
- FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class);
- FunctionArchive functionArchive = FunctionArchive.builder()
- .classLoader(mockedClassLoader)
- .build();
- when(mockedFunctionsManager.getFunction("exclamation")).thenReturn(functionArchive);
-
- when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager);
-
- when(mockedRuntimeFactory.externallyManaged()).thenReturn(true);
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
-
- FunctionConfig functionConfig = createDefaultFunctionConfig();
- functionConfig.setJar("builtin://exclamation");
-
- try (FileInputStream inputStream = new FileInputStream(getPulsarApiExamplesNar())) {
- resource.registerFunction(
- tenant,
- namespace,
- function,
- inputStream,
- mockedFormData,
- null,
- functionConfig,
- null);
- }
- }
-
- /*
- Externally managed runtime,
- uploadBuiltinSinksSources == true
- Make sure uploadFileToBookkeeper is called
- */
- @Test
- public void testRegisterFunctionSuccessK8sWithUpload() throws Exception {
- final String injectedErrMsg = "uploadFileToBookkeeper triggered";
- mockedWorkerService.getWorkerConfig().setUploadBuiltinSinksSources(true);
-
- mockStatic(WorkerUtils.class, ctx -> {
- ctx.when(() -> WorkerUtils.uploadFileToBookkeeper(
- anyString(),
- any(File.class),
- any(Namespace.class)))
- .thenThrow(new RuntimeException(injectedErrMsg));
-
- });
-
- NarClassLoader mockedClassLoader = mock(NarClassLoader.class);
- mockStatic(FunctionCommon.class, ctx -> {
- ctx.when(() -> FunctionCommon.getFunctionTypes(any(FunctionConfig.class), any(Class.class))).thenReturn(new Class[]{String.class, String.class});
- ctx.when(() -> FunctionCommon.convertRuntime(any(FunctionConfig.Runtime.class))).thenCallRealMethod();
- ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true);
- });
-
- doReturn(Function.class).when(mockedClassLoader).loadClass(anyString());
-
- FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class);
- FunctionArchive functionArchive = FunctionArchive.builder()
- .classLoader(mockedClassLoader)
- .build();
- when(mockedFunctionsManager.getFunction("exclamation")).thenReturn(functionArchive);
- when(mockedFunctionsManager.getFunctionArchive(any())).thenReturn(getPulsarApiExamplesNar().toPath());
-
- when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager);
-
- when(mockedRuntimeFactory.externallyManaged()).thenReturn(true);
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
-
- FunctionConfig functionConfig = createDefaultFunctionConfig();
- functionConfig.setJar("builtin://exclamation");
-
- try (FileInputStream inputStream = new FileInputStream(getPulsarApiExamplesNar())) {
- try {
- resource.registerFunction(
- tenant,
- namespace,
- function,
- inputStream,
- mockedFormData,
- null,
- functionConfig,
- null);
- Assert.fail();
- } catch (RuntimeException e) {
- Assert.assertEquals(e.getMessage(), injectedErrMsg);
- }
- }
- }
-
- //
- // Update Functions
- //
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided")
- public void testUpdateFunctionMissingTenant() throws Exception {
- try {
- testUpdateFunctionMissingArguments(
- null,
- namespace,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- "Tenant is not provided");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided")
- public void testUpdateFunctionMissingNamespace() throws Exception {
- try {
- testUpdateFunctionMissingArguments(
- tenant,
- null,
- function,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- "Namespace is not provided");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided")
- public void testUpdateFunctionMissingFunctionName() throws Exception {
- try {
- testUpdateFunctionMissingArguments(
- tenant,
- namespace,
- null,
- mockedInputStream,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- "Function name is not provided");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change")
- public void testUpdateFunctionMissingPackage() throws Exception {
- try {
- mockWorkerUtils();
- testUpdateFunctionMissingArguments(
- tenant,
- namespace,
- function,
- null,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- "Update contains no change");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change")
- public void testUpdateFunctionMissingInputTopic() throws Exception {
- try {
- mockWorkerUtils();
-
- testUpdateFunctionMissingArguments(
- tenant,
- namespace,
- function,
- null,
- null,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- className,
- parallelism,
- "Update contains no change");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change")
- public void testUpdateFunctionMissingClassName() throws Exception {
- try {
- mockWorkerUtils();
-
- testUpdateFunctionMissingArguments(
- tenant,
- namespace,
- function,
- null,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- null,
- parallelism,
- "Update contains no change");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test
- public void testUpdateFunctionChangedParallelism() throws Exception {
- try {
- mockWorkerUtils();
-
- testUpdateFunctionMissingArguments(
- tenant,
- namespace,
- function,
- null,
- topicsToSerDeClassName,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- null,
- parallelism + 1,
- null);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test
- public void testUpdateFunctionChangedInputs() throws Exception {
- mockWorkerUtils();
-
- testUpdateFunctionMissingArguments(
- tenant,
- namespace,
- function,
- null,
- topicsToSerDeClassName,
- mockedFormData,
- "DifferentOutput",
- outputSerdeClassName,
- null,
- parallelism,
- null);
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Input Topics cannot be altered")
- public void testUpdateFunctionChangedOutput() throws Exception {
- try {
- mockWorkerUtils();
-
- Map<String, String> someOtherInput = new HashMap<>();
- someOtherInput.put("DifferentTopic", TopicSchema.DEFAULT_SERDE);
- testUpdateFunctionMissingArguments(
- tenant,
- namespace,
- function,
- null,
- someOtherInput,
- mockedFormData,
- outputTopic,
- outputSerdeClassName,
- null,
- parallelism,
- "Input Topics cannot be altered");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- private void testUpdateFunctionMissingArguments(
- String tenant,
- String namespace,
- String function,
- InputStream inputStream,
- Map<String, String> topicsToSerDeClassName,
- FormDataContentDisposition details,
- String outputTopic,
- String outputSerdeClassName,
- String className,
- Integer parallelism,
- String expectedError) throws Exception {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- FunctionConfig functionConfig = new FunctionConfig();
- if (tenant != null) {
- functionConfig.setTenant(tenant);
- }
- if (namespace != null) {
- functionConfig.setNamespace(namespace);
- }
- if (function != null) {
- functionConfig.setName(function);
- }
- if (topicsToSerDeClassName != null) {
- functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
- }
- if (outputTopic != null) {
- functionConfig.setOutput(outputTopic);
- }
- if (outputSerdeClassName != null) {
- functionConfig.setOutputSerdeClassName(outputSerdeClassName);
- }
- if (className != null) {
- functionConfig.setClassName(className);
- }
- if (parallelism != null) {
- functionConfig.setParallelism(parallelism);
- }
- functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
-
- if (expectedError != null) {
- doThrow(new IllegalArgumentException(expectedError))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean());
- }
-
- resource.updateFunction(
- tenant,
- namespace,
- function,
- inputStream,
- details,
- null,
- functionConfig,
- null, null);
-
- }
-
- private void updateDefaultFunction() {
- updateDefaultFunctionWithPackageUrl(null);
- }
-
- private void updateDefaultFunctionWithPackageUrl(String packageUrl) {
- FunctionConfig functionConfig = new FunctionConfig();
- functionConfig.setTenant(tenant);
- functionConfig.setNamespace(namespace);
- functionConfig.setName(function);
- functionConfig.setClassName(className);
- functionConfig.setParallelism(parallelism);
- functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
- functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
- functionConfig.setOutput(outputTopic);
- functionConfig.setOutputSerdeClassName(outputSerdeClassName);
-
- resource.updateFunction(
- tenant,
- namespace,
- function,
- mockedInputStream,
- mockedFormData,
- packageUrl,
- functionConfig,
- null, null);
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist")
- public void testUpdateNotExistedFunction() {
- try {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
- updateDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure")
- public void testUpdateFunctionUploadFailure() throws Exception {
- try {
- mockWorkerUtils(ctx -> {
- ctx.when(() -> {
- WorkerUtils.uploadFileToBookkeeper(
- anyString(),
- any(File.class),
- any(Namespace.class));
-
- }).thenThrow(new IOException("upload failure"));
- ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod();
- });
-
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- updateDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR);
- throw re;
- }
- }
-
- @Test
- public void testUpdateFunctionSuccess() throws Exception {
- mockWorkerUtils();
-
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- updateDefaultFunction();
- }
-
- @Test
- public void testUpdateFunctionWithUrl() {
- Configurator.setRootLevel(Level.DEBUG);
-
- String fileLocation = FutureUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath();
- String filePackageUrl = "file://" + fileLocation;
-
- FunctionConfig functionConfig = new FunctionConfig();
- functionConfig.setOutput(outputTopic);
- functionConfig.setOutputSerdeClassName(outputSerdeClassName);
- functionConfig.setTenant(tenant);
- functionConfig.setNamespace(namespace);
- functionConfig.setName(function);
- functionConfig.setClassName(className);
- functionConfig.setParallelism(parallelism);
- functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
- functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
-
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- resource.updateFunction(
- tenant,
- namespace,
- function,
- null,
- null,
- filePackageUrl,
- functionConfig,
- null, null);
-
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register")
- public void testUpdateFunctionFailure() throws Exception {
- try {
- mockWorkerUtils();
-
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- doThrow(new IllegalArgumentException("function failed to register"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean());
-
- updateDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registeration interrupted")
- public void testUpdateFunctionInterrupted() throws Exception {
- try {
- mockWorkerUtils();
-
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- doThrow(new IllegalStateException("Function registeration interrupted"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean());
-
- updateDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR);
- throw re;
- }
- }
-
-
- @Test(timeOut = 20000)
- public void testUpdateFunctionSuccessWithPackageName() {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
- updateDefaultFunctionWithPackageUrl("function://public/default/test@v1");
- }
-
- @Test(timeOut = 20000)
- public void testUpdateFunctionFailedWithWrongPackageName() throws PulsarAdminException {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
- try {
- doThrow(new PulsarAdminException("package name is invalid"))
- .when(mockedPackages).download(anyString(), anyString());
- registerDefaultFunctionWithPackageUrl("function://");
- } catch (RestException e) {
- // expected exception
- assertEquals(e.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- }
- }
-
- //
- // deregister function
- //
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided")
- public void testDeregisterFunctionMissingTenant() {
- try {
-
- testDeregisterFunctionMissingArguments(
- null,
- namespace,
- function
- );
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided")
- public void testDeregisterFunctionMissingNamespace() {
- try {
- testDeregisterFunctionMissingArguments(
- tenant,
- null,
- function
- );
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided")
- public void testDeregisterFunctionMissingFunctionName() {
- try {
- testDeregisterFunctionMissingArguments(
- tenant,
- namespace,
- null
- );
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- private void testDeregisterFunctionMissingArguments(
- String tenant,
- String namespace,
- String function
- ) {
- resource.deregisterFunction(
- tenant,
- namespace,
- function,
- null);
- }
-
- private void deregisterDefaultFunction() {
- resource.deregisterFunction(
- tenant,
- namespace,
- function,
- null);
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist")
- public void testDeregisterNotExistedFunction() {
- try {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
- deregisterDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.NOT_FOUND);
- throw re;
- }
- }
-
- @Test
- public void testDeregisterFunctionSuccess() {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- deregisterDefaultFunction();
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to deregister")
- public void testDeregisterFunctionFailure() throws Exception {
- try {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- doThrow(new IllegalArgumentException("function failed to deregister"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean());
-
- deregisterDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
+public class FunctionApiV3ResourceTest extends AbstractFunctionApiResourceTest {
+ private FunctionsImpl resource;
+ @Override
+ protected void doSetup() {
+ super.doSetup();
+ this.resource = spy(new FunctionsImpl(() -> mockedWorkerService));
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function deregisteration interrupted")
- public void testDeregisterFunctionInterrupted() throws Exception {
- try {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- doThrow(new IllegalStateException("Function deregisteration interrupted"))
- .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean());
-
- deregisterDefaultFunction();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR);
- throw re;
- }
+ protected void registerFunction(String tenant, String namespace, String function, InputStream inputStream,
+ FormDataContentDisposition details, String functionPkgUrl, FunctionConfig functionConfig) {
+ resource.registerFunction(
+ tenant,
+ namespace,
+ function,
+ inputStream,
+ details,
+ functionPkgUrl,
+ functionConfig,
+ null);
+ }
+ protected void updateFunction(String tenant,
+ String namespace,
+ String functionName,
+ InputStream uploadedInputStream,
+ FormDataContentDisposition fileDetail,
+ String functionPkgUrl,
+ FunctionConfig functionConfig,
+ AuthenticationParameters authParams,
+ UpdateOptionsImpl updateOptions) {
+ resource.updateFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail, functionPkgUrl,
+ functionConfig, authParams, updateOptions);
}
- //
- // Get Function Info
- //
+ protected StreamingOutput downloadFunction(String tenant, String namespace, String componentName,
+ AuthenticationParameters authParams) {
+ return resource.downloadFunction(tenant, namespace, componentName, authParams);
+ }
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided")
- public void testGetFunctionMissingTenant() {
- try {
- testGetFunctionMissingArguments(
- null,
- namespace,
- function
- );
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
+ protected File downloadFunction(final String path, final AuthenticationParameters authParams) throws IOException {
+ StreamingOutput streamingOutput = resource.downloadFunction(path, authParams);
+ File pkgFile = File.createTempFile("testpkg", "nar");
+ try(OutputStream output = new FileOutputStream(pkgFile)) {
+ streamingOutput.write(output);
}
+ return pkgFile;
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided")
- public void testGetFunctionMissingNamespace() {
- try {
- testGetFunctionMissingArguments(
+ protected void testDeregisterFunctionMissingArguments(
+ String tenant,
+ String namespace,
+ String function
+ ) {
+ resource.deregisterFunction(
tenant,
- null,
- function
- );
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
+ namespace,
+ function,
+ null);
}
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided")
- public void testGetFunctionMissingFunctionName() {
- try {
- testGetFunctionMissingArguments(
+ protected void deregisterDefaultFunction() {
+ resource.deregisterFunction(
tenant,
namespace,
- null
- );
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
+ function,
+ null);
}
- private void testGetFunctionMissingArguments(
+ protected void testGetFunctionMissingArguments(
String tenant,
String namespace,
String function
) {
resource.getFunctionInfo(
- tenant,
- namespace,
- function,null
+ tenant,
+ namespace,
+ function, null
);
}
- private FunctionConfig getDefaultFunctionInfo() {
+ protected FunctionConfig getDefaultFunctionInfo() {
return resource.getFunctionInfo(
- tenant,
- namespace,
- function,
- null
- );
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist")
- public void testGetNotExistedFunction() {
- try {
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
- getDefaultFunctionInfo();
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.NOT_FOUND);
- throw re;
- }
- }
-
- @Test
- public void testGetFunctionSuccess() {
- mockInstanceUtils();
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
-
- SinkSpec sinkSpec = SinkSpec.newBuilder()
- .setTopic(outputTopic)
- .setSerDeClassName(outputSerdeClassName).build();
- FunctionDetails functionDetails = FunctionDetails.newBuilder()
- .setClassName(className)
- .setSink(sinkSpec)
- .setAutoAck(true)
- .setName(function)
- .setNamespace(namespace)
- .setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE)
- .setTenant(tenant)
- .setParallelism(parallelism)
- .setSource(SourceSpec.newBuilder().setSubscriptionType(subscriptionType)
- .putAllTopicsToSerDeClassName(topicsToSerDeClassName)).build();
- FunctionMetaData metaData = FunctionMetaData.newBuilder()
- .setCreateTime(System.currentTimeMillis())
- .setFunctionDetails(functionDetails)
- .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("/path/to/package"))
- .setVersion(1234)
- .build();
- when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData);
-
- FunctionConfig functionConfig = getDefaultFunctionInfo();
- assertEquals(
- FunctionConfigUtils.convertFromDetails(functionDetails),
- functionConfig);
- }
-
- //
- // List Functions
- //
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided")
- public void testListFunctionsMissingTenant() {
- try {
- testListFunctionsMissingArguments(
- null,
- namespace
- );
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
- }
-
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided")
- public void testListFunctionsMissingNamespace() {
- try {
- testListFunctionsMissingArguments(
tenant,
+ namespace,
+ function,
null
- );
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST);
- throw re;
- }
+ );
}
- private void testListFunctionsMissingArguments(
+ protected void testListFunctionsMissingArguments(
String tenant,
String namespace
) {
resource.listFunctions(
- tenant,
- namespace,null
+ tenant,
+ namespace, null
);
}
- private List<String> listDefaultFunctions() {
+ protected List<String> listDefaultFunctions() {
return resource.listFunctions(
- tenant,
- namespace,null
+ tenant,
+ namespace, null
);
}
@Test
- public void testListFunctionsSuccess() {
- mockInstanceUtils();
- final List<String> functions = Lists.newArrayList("test-1", "test-2");
- final List<FunctionMetaData> metaDataList = new LinkedList<>();
- FunctionMetaData functionMetaData1 = FunctionMetaData.newBuilder().setFunctionDetails(
- FunctionDetails.newBuilder().setName("test-1").build()
- ).build();
- FunctionMetaData functionMetaData2 = FunctionMetaData.newBuilder().setFunctionDetails(
- FunctionDetails.newBuilder().setName("test-2").build()
- ).build();
- metaDataList.add(functionMetaData1);
- metaDataList.add(functionMetaData2);
- when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(metaDataList);
-
- List<String> functionList = listDefaultFunctions();
- assertEquals(functions, functionList);
- }
-
- @Test
- public void testOnlyGetSources() {
- List<String> functions = Lists.newArrayList("test-2");
- List<FunctionMetaData> functionMetaDataList = new LinkedList<>();
- FunctionMetaData f1 = FunctionMetaData.newBuilder().setFunctionDetails(
- FunctionDetails.newBuilder()
- .setName("test-1")
- .setComponentType(FunctionDetails.ComponentType.SOURCE)
- .build()).build();
- functionMetaDataList.add(f1);
- FunctionMetaData f2 = FunctionMetaData.newBuilder().setFunctionDetails(
- FunctionDetails.newBuilder()
- .setName("test-2")
- .setComponentType(FunctionDetails.ComponentType.FUNCTION)
- .build()).build();
- functionMetaDataList.add(f2);
- FunctionMetaData f3 = FunctionMetaData.newBuilder().setFunctionDetails(
- FunctionDetails.newBuilder()
- .setName("test-3")
- .setComponentType(FunctionDetails.ComponentType.SINK)
- .build()).build();
- functionMetaDataList.add(f3);
- when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(functionMetaDataList);
-
- List<String> functionList = listDefaultFunctions();
- assertEquals(functions, functionList);
- }
-
- @Test
- public void testDownloadFunctionHttpUrl() throws Exception {
- String jarHttpUrl =
- "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar";
- String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath();
-
- StreamingOutput streamOutput = resource.downloadFunction(jarHttpUrl, null);
- File pkgFile = new File(testDir, UUID.randomUUID().toString());
- OutputStream output = new FileOutputStream(pkgFile);
- streamOutput.write(output);
- Assert.assertTrue(pkgFile.exists());
- pkgFile.delete();
- }
-
- @Test
- public void testDownloadFunctionFile() throws Exception {
- URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
- File file = Paths.get(fileUrl.toURI()).toFile();
- String fileLocation = file.getAbsolutePath().replace('\\', '/');
- String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath();
-
- StreamingOutput streamOutput = resource.downloadFunction("file:///" + fileLocation, null);
- File pkgFile = new File(testDir, UUID.randomUUID().toString());
- OutputStream output = new FileOutputStream(pkgFile);
- streamOutput.write(output);
- Assert.assertTrue(pkgFile.exists());
- Assert.assertEquals(file.length(), pkgFile.length());
- pkgFile.delete();
- }
-
- @Test
- public void testDownloadFunctionBuiltinConnector() throws Exception {
- URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
- File file = Paths.get(fileUrl.toURI()).toFile();
- String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath();
-
+ public void testDownloadFunctionBuiltinConnectorByName() throws Exception {
+ File file = getPulsarApiExamplesNar();
WorkerConfig config = new WorkerConfig()
- .setUploadBuiltinSinksSources(false);
+ .setUploadBuiltinSinksSources(false);
when(mockedWorkerService.getWorkerConfig()).thenReturn(config);
- Connector connector = Connector.builder().archivePath(file.toPath()).build();
- ConnectorsManager connectorsManager = mock(ConnectorsManager.class);
- when(connectorsManager.getConnector("cassandra")).thenReturn(connector);
- when(mockedWorkerService.getConnectorsManager()).thenReturn(connectorsManager);
-
- StreamingOutput streamOutput = resource.downloadFunction("builtin://cassandra", null);
-
- File pkgFile = new File(testDir, UUID.randomUUID().toString());
- OutputStream output = new FileOutputStream(pkgFile);
- streamOutput.write(output);
- output.flush();
- output.close();
- Assert.assertTrue(pkgFile.exists());
- Assert.assertTrue(pkgFile.exists());
- Assert.assertEquals(file.length(), pkgFile.length());
- pkgFile.delete();
- }
-
- @Test
- public void testDownloadFunctionBuiltinFunction() throws Exception {
- URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
- File file = Paths.get(fileUrl.toURI()).toFile();
- String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath();
-
- WorkerConfig config = new WorkerConfig()
- .setUploadBuiltinSinksSources(false);
- when(mockedWorkerService.getWorkerConfig()).thenReturn(config);
+ when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
- FunctionsManager functionsManager = mock(FunctionsManager.class);
- FunctionArchive functionArchive = FunctionArchive.builder().archivePath(file.toPath()).build();
- when(functionsManager.getFunction("exclamation")).thenReturn(functionArchive);
- when(mockedWorkerService.getConnectorsManager()).thenReturn(mock(ConnectorsManager.class));
- when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager);
+ Function.FunctionMetaData metaData = Function.FunctionMetaData.newBuilder()
+ .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("builtin://cassandra"))
+ .setFunctionDetails(Function.FunctionDetails.newBuilder().setComponentType(Function.FunctionDetails.ComponentType.SINK))
+ .build();
+ when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData);
- StreamingOutput streamOutput = resource.downloadFunction("builtin://exclamation", null);
+ registerBuiltinConnector("cassandra", file);
- File pkgFile = new File(testDir, UUID.randomUUID().toString());
+ StreamingOutput streamOutput = downloadFunction(tenant, namespace, function,
+ AuthenticationParameters.builder().build());
+ File pkgFile = File.createTempFile("testpkg", "nar");
OutputStream output = new FileOutputStream(pkgFile);
streamOutput.write(output);
- output.flush();
- output.close();
Assert.assertTrue(pkgFile.exists());
Assert.assertEquals(file.length(), pkgFile.length());
pkgFile.delete();
}
@Test
- public void testDownloadFunctionBuiltinConnectorByName() throws Exception {
- URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
- File file = Paths.get(fileUrl.toURI()).toFile();
- String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath();
+ public void testDownloadFunctionBuiltinFunctionByName() throws Exception {
+ File file = getPulsarApiExamplesNar();
WorkerConfig config = new WorkerConfig()
- .setUploadBuiltinSinksSources(false);
+ .setUploadBuiltinSinksSources(false);
when(mockedWorkerService.getWorkerConfig()).thenReturn(config);
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
- FunctionMetaData metaData = FunctionMetaData.newBuilder()
- .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("builtin://cassandra"))
- .setFunctionDetails(FunctionDetails.newBuilder().setComponentType(FunctionDetails.ComponentType.SINK))
+ Function.FunctionMetaData metaData = Function.FunctionMetaData.newBuilder()
+ .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("builtin://exclamation"))
+ .setFunctionDetails(
+ Function.FunctionDetails.newBuilder().setComponentType(Function.FunctionDetails.ComponentType.FUNCTION))
.build();
when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData);
- Connector connector = Connector.builder().archivePath(file.toPath()).build();
- ConnectorsManager connectorsManager = mock(ConnectorsManager.class);
- when(connectorsManager.getConnector("cassandra")).thenReturn(connector);
- when(mockedWorkerService.getConnectorsManager()).thenReturn(connectorsManager);
+ registerBuiltinFunction("exclamation", file);
- StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function,
+ StreamingOutput streamOutput = downloadFunction(tenant, namespace, function,
AuthenticationParameters.builder().build());
- File pkgFile = new File(testDir, UUID.randomUUID().toString());
+ File pkgFile = File.createTempFile("testpkg", "nar");
OutputStream output = new FileOutputStream(pkgFile);
streamOutput.write(output);
Assert.assertTrue(pkgFile.exists());
@@ -1715,47 +210,58 @@ public class FunctionApiV3ResourceTest {
pkgFile.delete();
}
- @Test
- public void testDownloadFunctionBuiltinFunctionByName() throws Exception {
- URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
- File file = Paths.get(fileUrl.toURI()).toFile();
- String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath();
- WorkerConfig config = new WorkerConfig()
- .setUploadBuiltinSinksSources(false);
- when(mockedWorkerService.getWorkerConfig()).thenReturn(config);
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't"
+ + " exist")
+ public void testGetNotExistedFunction() throws IOException {
+ try {
+ when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
+ getDefaultFunctionInfo();
+ } catch (RestException re) {
+ assertEquals(re.getResponse().getStatusInfo(), Response.Status.NOT_FOUND);
+ throw re;
+ }
+ }
+ @Test
+ public void testGetFunctionSuccess() throws IOException {
+ mockInstanceUtils();
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
- FunctionMetaData metaData = FunctionMetaData.newBuilder()
- .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("builtin://exclamation"))
- .setFunctionDetails(FunctionDetails.newBuilder().setComponentType(FunctionDetails.ComponentType.FUNCTION))
- .build();
+ Function.SinkSpec sinkSpec = Function.SinkSpec.newBuilder()
+ .setTopic(outputTopic)
+ .setSerDeClassName(outputSerdeClassName).build();
+ Function.FunctionDetails functionDetails = Function.FunctionDetails.newBuilder()
+ .setClassName(className)
+ .setSink(sinkSpec)
+ .setAutoAck(true)
+ .setName(function)
+ .setNamespace(namespace)
+ .setProcessingGuarantees(Function.ProcessingGuarantees.ATMOST_ONCE)
+ .setTenant(tenant)
+ .setParallelism(parallelism)
+ .setSource(Function.SourceSpec.newBuilder().setSubscriptionType(subscriptionType)
+ .putAllTopicsToSerDeClassName(topicsToSerDeClassName)).build();
+ Function.FunctionMetaData metaData = Function.FunctionMetaData.newBuilder()
+ .setCreateTime(System.currentTimeMillis())
+ .setFunctionDetails(functionDetails)
+ .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/path/to/package"))
+ .setVersion(1234)
+ .build();
when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData);
- FunctionsManager functionsManager = mock(FunctionsManager.class);
- FunctionArchive functionArchive = FunctionArchive.builder().archivePath(file.toPath()).build();
- when(functionsManager.getFunction("exclamation")).thenReturn(functionArchive);
- when(mockedWorkerService.getConnectorsManager()).thenReturn(mock(ConnectorsManager.class));
- when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager);
-
- StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function,
- AuthenticationParameters.builder().build());
- File pkgFile = new File(testDir, UUID.randomUUID().toString());
- OutputStream output = new FileOutputStream(pkgFile);
- streamOutput.write(output);
- Assert.assertTrue(pkgFile.exists());
- Assert.assertEquals(file.length(), pkgFile.length());
- pkgFile.delete();
+ FunctionConfig functionConfig = getDefaultFunctionInfo();
+ assertEquals(
+ FunctionConfigUtils.convertFromDetails(functionDetails),
+ functionConfig);
}
- @Test
- public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception {
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function language runtime is "
+ + "either not set or cannot be determined")
+ public void testCreateFunctionWithoutSettingRuntime() throws Exception {
Configurator.setRootLevel(Level.DEBUG);
- URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
- File file = Paths.get(fileUrl.toURI()).toFile();
- String fileLocation = file.getAbsolutePath().replace('\\', '/');
- String filePackageUrl = "file:///" + fileLocation;
+ File file = getPulsarApiExamplesNar();
+ String filePackageUrl = file.toURI().toString();
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
FunctionConfig functionConfig = new FunctionConfig();
@@ -1764,82 +270,78 @@ public class FunctionApiV3ResourceTest {
functionConfig.setName(function);
functionConfig.setClassName(className);
functionConfig.setParallelism(parallelism);
- functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
functionConfig.setOutput(outputTopic);
functionConfig.setOutputSerdeClassName(outputSerdeClassName);
- resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig, null);
+ registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig);
}
- @Test
- public void testRegisterFunctionWithConflictingFields() throws Exception {
- Configurator.setRootLevel(Level.DEBUG);
- String actualTenant = "DIFFERENT_TENANT";
- String actualNamespace = "DIFFERENT_NAMESPACE";
- String actualName = "DIFFERENT_NAME";
- this.namespaceList.add(actualTenant + "/" + actualNamespace);
+ @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function config is not provided")
+ public void testMissingFunctionConfig() throws IOException {
+ registerFunction(tenant, namespace, function, mockedInputStream, mockedFormData, null, null);
+ }
- URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
- File file = Paths.get(fileUrl.toURI()).toFile();
- String fileLocation = file.getAbsolutePath().replace('\\', '/');
- String filePackageUrl = "file:///" + fileLocation;
- when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true);
- when(mockedManager.containsFunction(eq(actualTenant), eq(actualNamespace), eq(actualName))).thenReturn(false);
+ /*
+ Externally managed runtime,
+ uploadBuiltinSinksSources == false
+ Make sure uploadFileToBookkeeper is not called
+ */
+ @Test
+ public void testRegisterFunctionSuccessK8sNoUpload() throws Exception {
+ mockedWorkerService.getWorkerConfig().setUploadBuiltinSinksSources(false);
- FunctionConfig functionConfig = new FunctionConfig();
- functionConfig.setTenant(tenant);
- functionConfig.setNamespace(namespace);
- functionConfig.setName(function);
- functionConfig.setClassName(className);
- functionConfig.setParallelism(parallelism);
- functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
- functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
- functionConfig.setOutput(outputTopic);
- functionConfig.setOutputSerdeClassName(outputSerdeClassName);
- resource.registerFunction(actualTenant, actualNamespace, actualName, null, null, filePackageUrl, functionConfig,
- null);
- }
+ mockStatic(WorkerUtils.class, ctx -> {
+ ctx.when(() -> WorkerUtils.uploadFileToBookkeeper(
+ anyString(),
+ any(File.class),
+ any(Namespace.class)))
+ .thenThrow(new RuntimeException("uploadFileToBookkeeper triggered"));
- @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function language runtime is either not set or cannot be determined")
- public void testCreateFunctionWithoutSettingRuntime() throws Exception {
- Configurator.setRootLevel(Level.DEBUG);
+ });
- URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml");
- File file = Paths.get(fileUrl.toURI()).toFile();
- String fileLocation = file.getAbsolutePath().replace('\\', '/');
- String filePackageUrl = "file:///" + fileLocation;
+ registerBuiltinFunction("exclamation", getPulsarApiExamplesNar());
+ when(mockedRuntimeFactory.externallyManaged()).thenReturn(true);
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
- FunctionConfig functionConfig = new FunctionConfig();
- functionConfig.setTenant(tenant);
- functionConfig.setNamespace(namespace);
- functionConfig.setName(function);
- functionConfig.setClassName(className);
- functionConfig.setParallelism(parallelism);
- functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
- functionConfig.setOutput(outputTopic);
- functionConfig.setOutputSerdeClassName(outputSerdeClassName);
- resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig, null);
+ FunctionConfig functionConfig = createDefaultFunctionConfig();
+ functionConfig.setJar("builtin://exclamation");
+ registerFunction(tenant, namespace, function, null, mockedFormData, null, functionConfig);
}
- public static FunctionConfig createDefaultFunctionConfig() {
- FunctionConfig functionConfig = new FunctionConfig();
- functionConfig.setTenant(tenant);
- functionConfig.setNamespace(namespace);
- functionConfig.setName(function);
- functionConfig.setClassName(className);
- functionConfig.setParallelism(parallelism);
- functionConfig.setCustomSerdeInputs(topicsToSerDeClassName);
- functionConfig.setOutput(outputTopic);
- functionConfig.setOutputSerdeClassName(outputSerdeClassName);
- functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
- return functionConfig;
- }
+ /*
+ Externally managed runtime,
+ uploadBuiltinSinksSources == true
+ Make sure uploadFileToBookkeeper is called
+ */
+ @Test
+ public void testRegisterFunctionSuccessK8sWithUpload() throws Exception {
+ final String injectedErrMsg = "uploadFileToBookkeeper triggered";
+ mockedWorkerService.getWorkerConfig().setUploadBuiltinSinksSources(true);
+
+ mockStatic(WorkerUtils.class, ctx -> {
+ ctx.when(() -> WorkerUtils.uploadFileToBookkeeper(
+ anyString(),
+ any(File.class),
+ any(Namespace.class)))
+ .thenThrow(new RuntimeException(injectedErrMsg));
+
+ });
+
+ registerBuiltinFunction("exclamation", getPulsarApiExamplesNar());
+ when(mockedRuntimeFactory.externallyManaged()).thenReturn(true);
+ when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false);
- public static FunctionDetails createDefaultFunctionDetails() {
FunctionConfig functionConfig = createDefaultFunctionConfig();
- return FunctionConfigUtils.convert(functionConfig, (ClassLoader) null);
+ functionConfig.setJar("builtin://exclamation");
+
+ try {
+ registerFunction(tenant, namespace, function, null, mockedFormData, null, functionConfig);
+ Assert.fail();
+ } catch (RuntimeException e) {
+ Assert.assertEquals(e.getMessage(), injectedErrMsg);
+ }
}
+
}
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java
index a061c23eaa6..a9927514efd 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java
@@ -22,232 +22,73 @@ import static org.apache.pulsar.functions.proto.Function.ProcessingGuarantees.AT
import static org.apache.pulsar.functions.source.TopicSchema.DEFAULT_SERDE;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyString;
-import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.eq;
-import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
import com.google.common.collect.Lists;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
-import java.util.function.Consumer;
import javax.ws.rs.core.Response;
import org.apache.distributedlog.api.namespace.Namespace;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.core.config.Configurator;
import org.apache.pulsar.broker.authentication.AuthenticationParameters;
-import org.apache.pulsar.client.admin.Functions;
-import org.apache.pulsar.client.admin.Namespaces;
-import org.apache.pulsar.client.admin.Packages;
-import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.client.admin.Tenants;
-import org.apache.pulsar.common.functions.FunctionConfig;
+import org.apache.pulsar.common.functions.UpdateOptionsImpl;
import org.apache.pulsar.common.functions.Utils;
import org.apache.pulsar.common.io.SinkConfig;
-import org.apache.pulsar.common.nar.NarClassLoader;
-import org.apache.pulsar.common.policies.data.TenantInfoImpl;
-import org.apache.pulsar.common.util.ClassLoaderUtils;
import org.apache.pulsar.common.util.RestException;
import org.apache.pulsar.functions.api.utils.IdentityFunction;
import org.apache.pulsar.functions.instance.InstanceUtils;
import org.apache.pulsar.functions.proto.Function;
import org.apache.pulsar.functions.proto.Function.FunctionDetails;
import org.apache.pulsar.functions.proto.Function.FunctionMetaData;
-import org.apache.pulsar.functions.runtime.RuntimeFactory;
-import org.apache.pulsar.functions.utils.FunctionCommon;
import org.apache.pulsar.functions.utils.SinkConfigUtils;
-import org.apache.pulsar.functions.utils.io.Connector;
-import org.apache.pulsar.functions.utils.io.ConnectorUtils;
-import org.apache.pulsar.functions.worker.ConnectorsManager;
-import org.apache.pulsar.functions.worker.FunctionMetaDataManager;
-import org.apache.pulsar.functions.worker.FunctionRuntimeManager;
-import org.apache.pulsar.functions.worker.LeaderService;
-import org.apache.pulsar.functions.worker.PulsarWorkerService;
-import org.apache.pulsar.functions.worker.WorkerConfig;
import org.apache.pulsar.functions.worker.WorkerUtils;
-import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory;
import org.apache.pulsar.functions.worker.rest.api.SinksImpl;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
/**
* Unit test of {@link SinksApiV3Resource}.
*/
-public class SinkApiV3ResourceTest {
+public class SinkApiV3ResourceTest extends AbstractFunctionsResourceTest {
- private static final String tenant = "test-tenant";
- private static final String namespace = "test-namespace";
private static final String sink = "test-sink";
- private static final Map<String, String> topicsToSerDeClassName = new HashMap<>();
-
- static {
- topicsToSerDeClassName.put("test_src", DEFAULT_SERDE);
- }
-
- private static final String subscriptionName = "test-subscription";
- private static final String CASSANDRA_STRING_SINK = "org.apache.pulsar.io.cassandra.CassandraStringSink";
- private static final int parallelism = 1;
-
- private PulsarWorkerService mockedWorkerService;
- private PulsarAdmin mockedPulsarAdmin;
- private Tenants mockedTenants;
- private Namespaces mockedNamespaces;
- private Functions mockedFunctions;
- private TenantInfoImpl mockedTenantInfo;
- private List<String> namespaceList = new LinkedList<>();
- private FunctionMetaDataManager mockedManager;
- private FunctionRuntimeManager mockedFunctionRunTimeManager;
- private RuntimeFactory mockedRuntimeFactory;
- private Namespace mockedNamespace;
+
private SinksImpl resource;
- private InputStream mockedInputStream;
- private FormDataContentDisposition mockedFormData;
- private FunctionMetaData mockedFunctionMetaData;
- private LeaderService mockedLeaderService;
- private Packages mockedPackages;
- private PulsarFunctionTestTemporaryDirectory tempDirectory;
- private static Map<String, MockedStatic> mockStaticContexts = new HashMap<>();
-
- private static final String SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH = "pulsar-io-cassandra.nar.path";
-
- public static File getPulsarIOCassandraNar() {
- return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH)
- , "pulsar-io-cassandra.nar file location must be specified with "
- + SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH + " system property"));
- }
-
- private static final String SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH = "pulsar-io-twitter.nar.path";
-
- public static File getPulsarIOTwitterNar() {
- return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH)
- , "pulsar-io-twitter.nar file location must be specified with "
- + SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH + " system property"));
- }
-
- private static final String SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH = "pulsar-io-invalid.nar.path";
-
- public static File getPulsarIOInvalidNar() {
- return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH)
- , "invalid nar file location must be specified with "
- + SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH + " system property"));
- }
-
- @BeforeMethod
- public void setup() throws Exception {
- this.mockedManager = mock(FunctionMetaDataManager.class);
- this.mockedFunctionRunTimeManager = mock(FunctionRuntimeManager.class);
- this.mockedRuntimeFactory = mock(RuntimeFactory.class);
- this.mockedInputStream = mock(InputStream.class);
- this.mockedNamespace = mock(Namespace.class);
- this.mockedFormData = mock(FormDataContentDisposition.class);
- when(mockedFormData.getFileName()).thenReturn("test");
- this.mockedTenantInfo = mock(TenantInfoImpl.class);
- this.mockedPulsarAdmin = mock(PulsarAdmin.class);
- this.mockedTenants = mock(Tenants.class);
- this.mockedNamespaces = mock(Namespaces.class);
- this.mockedFunctions = mock(Functions.class);
- this.mockedLeaderService = mock(LeaderService.class);
- this.mockedPackages = mock(Packages.class);
- namespaceList.add(tenant + "/" + namespace);
-
- this.mockedWorkerService = mock(PulsarWorkerService.class);
- when(mockedWorkerService.getFunctionMetaDataManager()).thenReturn(mockedManager);
- when(mockedWorkerService.getLeaderService()).thenReturn(mockedLeaderService);
- when(mockedWorkerService.getFunctionRuntimeManager()).thenReturn(mockedFunctionRunTimeManager);
- when(mockedFunctionRunTimeManager.getRuntimeFactory()).thenReturn(mockedRuntimeFactory);
- when(mockedWorkerService.getDlogNamespace()).thenReturn(mockedNamespace);
- when(mockedWorkerService.isInitialized()).thenReturn(true);
- when(mockedWorkerService.getBrokerAdmin()).thenReturn(mockedPulsarAdmin);
- when(mockedWorkerService.getFunctionAdmin()).thenReturn(mockedPulsarAdmin);
- when(mockedPulsarAdmin.tenants()).thenReturn(mockedTenants);
- when(mockedPulsarAdmin.namespaces()).thenReturn(mockedNamespaces);
- when(mockedPulsarAdmin.functions()).thenReturn(mockedFunctions);
- when(mockedPulsarAdmin.packages()).thenReturn(mockedPackages);
- when(mockedTenants.getTenantInfo(any())).thenReturn(mockedTenantInfo);
- when(mockedNamespaces.getNamespaces(any())).thenReturn(namespaceList);
- when(mockedLeaderService.isLeader()).thenReturn(true);
- doAnswer(invocationOnMock -> {
- Files.copy(getPulsarIOCassandraNar().toPath(), Paths.get(invocationOnMock.getArgument(1, String.class)),
- StandardCopyOption.REPLACE_EXISTING);
- return null;
- }).when(mockedPackages).download(any(), any());
-
- // worker config
- WorkerConfig workerConfig = new WorkerConfig()
- .setWorkerId("test")
- .setWorkerPort(8080)
- .setFunctionMetadataTopicName("pulsar/functions")
- .setNumFunctionPackageReplicas(3)
- .setPulsarServiceUrl("pulsar://localhost:6650/");
... 1147 lines suppressed ...