You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by si...@apache.org on 2018/03/07 05:56:23 UTC
[incubator-pulsar] branch master updated: Shade
pulsar-functions-runtime instead of pulsar-functions-worker (#1351)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 0ac6b9a Shade pulsar-functions-runtime instead of pulsar-functions-worker (#1351)
0ac6b9a is described below
commit 0ac6b9aaf49ce9e3d0fcd5a856cdec48a2fdd546
Author: Sijie Guo <gu...@gmail.com>
AuthorDate: Tue Mar 6 21:56:20 2018 -0800
Shade pulsar-functions-runtime instead of pulsar-functions-worker (#1351)
---
all/pom.xml | 9 ++
all/src/assemble/bin.xml | 2 +-
bin/pulsar | 2 +-
bin/pulsar-admin | 2 +-
pom.xml | 9 +-
pulsar-broker/pom.xml | 94 ++------------
.../apache/pulsar/admin/cli/CmdFunctionsTest.java | 4 +-
pulsar-client-tools/pom.xml | 77 +-----------
.../org/apache/pulsar/admin/cli/CmdFunctions.java | 6 +-
pulsar-common/pom.xml | 11 +-
pulsar-functions/pom.xml | 2 +
.../runtime-all}/pom.xml | 138 +++++++++++----------
.../{worker-shaded => runtime-shaded}/pom.xml | 89 +++++++++++--
pulsar-functions/runtime/pom.xml | 34 -----
.../pulsar/functions/runtime/ProcessRuntime.java | 29 +++++
pulsar-functions/utils/pom.xml | 14 +++
pulsar-functions/worker-shaded/pom.xml | 34 +++++
pulsar-functions/worker/pom.xml | 80 +++++++++++-
.../functions/worker/FunctionAssignmentTailer.java | 3 +-
.../worker/FunctionMetaDataTopicTailer.java | 3 +-
.../functions/worker/SchedulerManagerTest.java | 22 ++--
21 files changed, 367 insertions(+), 297 deletions(-)
diff --git a/all/pom.xml b/all/pom.xml
index 6273f4b..7368700 100644
--- a/all/pom.xml
+++ b/all/pom.xml
@@ -113,6 +113,15 @@
<!-- make sure the api examples are compiled before assembly -->
<scope>provided</scope>
</dependency>
+
+ <!-- runtime-all -->
+ <dependency>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-runtime-all</artifactId>
+ <version>${project.version}</version>
+ <!-- make sure the api examples are compiled before assembly -->
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/all/src/assemble/bin.xml b/all/src/assemble/bin.xml
index 3a994ac..e74b684 100644
--- a/all/src/assemble/bin.xml
+++ b/all/src/assemble/bin.xml
@@ -73,7 +73,7 @@
<fileMode>644</fileMode>
</file>
<file>
- <source>${basedir}/../pulsar-functions/runtime/target/java-instance.jar</source>
+ <source>${basedir}/../pulsar-functions/runtime-all/target/java-instance.jar</source>
<destName>java-instance.jar</destName>
<outputDirectory>instances</outputDirectory>
</file>
diff --git a/bin/pulsar b/bin/pulsar
index cafc359..0bd2228 100755
--- a/bin/pulsar
+++ b/bin/pulsar
@@ -78,7 +78,7 @@ fi
# find the java instance location
if [ ! -f "${JAVA_INSTANCE_JAR}" ]; then
# didn't find a released jar, then search the built jar
- BUILT_JAVA_INSTANCE_JAR="${FUNCTIONS_HOME}/runtime/target/java-instance.jar"
+ BUILT_JAVA_INSTANCE_JAR="${FUNCTIONS_HOME}/runtime-all/target/java-instance.jar"
if [ -z "${BUILT_JAVA_INSTANCE_JAR}" ]; then
echo "\nCouldn't find pulsar-functions java instance jar.";
echo "Make sure you've run 'mvn package'\n";
diff --git a/bin/pulsar-admin b/bin/pulsar-admin
index 2e49225..c8b41da 100755
--- a/bin/pulsar-admin
+++ b/bin/pulsar-admin
@@ -117,7 +117,7 @@ OPTS="$OPTS -Dpulsar.routing.appender.default=$PULSAR_ROUTING_APPENDER_DEFAULT"
# find the java instance location
if [ ! -f "${JAVA_INSTANCE_JAR}" ]; then
# didn't find a released jar, then search the built jar
- BUILT_JAVA_INSTANCE_JAR="${FUNCTIONS_HOME}/runtime/target/java-instance.jar"
+ BUILT_JAVA_INSTANCE_JAR="${FUNCTIONS_HOME}/runtime-all/target/java-instance.jar"
if [ -f "${BUILT_JAVA_INSTANCE_JAR}" ]; then
JAVA_INSTANCE_JAR=${BUILT_JAVA_INSTANCE_JAR}
else
diff --git a/pom.xml b/pom.xml
index 78c6641..081fcdb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -320,6 +320,12 @@ flexible messaging model and an intuitive client API.</description>
<dependency>
<groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ <version>2.0.7.Final</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.netty</groupId>
<artifactId>netty</artifactId>
<version>3.10.1.Final</version>
</dependency>
@@ -731,9 +737,8 @@ flexible messaging model and an intuitive client API.</description>
<dependency>
<!-- We use MockedBookKeeper in many unit tests -->
<groupId>org.apache.bookkeeper</groupId>
- <artifactId>bookkeeper-server</artifactId>
+ <artifactId>bookkeeper-server-tests-shaded</artifactId>
<version>${bookkeeper.version}</version>
- <classifier>tests</classifier>
<scope>test</scope>
<exclusions>
<exclusion>
diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml
index abbc831..abead55 100644
--- a/pulsar-broker/pom.xml
+++ b/pulsar-broker/pom.xml
@@ -56,6 +56,18 @@
<dependency>
<groupId>${project.groupId}</groupId>
+ <artifactId>pulsar-common</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.bookkeeper</groupId>
+ <artifactId>circe-checksum</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>${project.groupId}</groupId>
<artifactId>pulsar-client-original</artifactId>
<version>${project.version}</version>
</dependency>
@@ -123,88 +135,8 @@
<dependency>
<groupId>${project.groupId}</groupId>
- <artifactId>pulsar-functions-worker-shaded</artifactId>
+ <artifactId>pulsar-functions-worker</artifactId>
<version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-lite</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf.nano</groupId>
- <artifactId>protobuf-javanano</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf-lite</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf-nano</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-utils</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-metrics</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-instance</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-worker</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>net.jodah</groupId>
- <artifactId>typetools</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-api</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.glassfish.jersey.media</groupId>
- <artifactId>jersey-media-multipart</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.distributedlog</groupId>
- <artifactId>distributedlog-core-shaded</artifactId>
</dependency>
<!-- functions related dependencies (end) -->
diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java
index e57aa84..2cd664a 100644
--- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java
+++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java
@@ -31,8 +31,8 @@ import static org.powermock.api.mockito.PowerMockito.mockStatic;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNull;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufUtil;
+import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBuf;
+import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBufUtil;
import java.io.File;
import java.net.URI;
import java.util.concurrent.atomic.AtomicReference;
diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml
index 693eca1..5779b78 100644
--- a/pulsar-client-tools/pom.xml
+++ b/pulsar-client-tools/pom.xml
@@ -76,83 +76,8 @@
<dependency>
<groupId>${project.groupId}</groupId>
- <artifactId>pulsar-functions-worker-shaded</artifactId>
+ <artifactId>pulsar-functions-worker</artifactId>
<version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-lite</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf.nano</groupId>
- <artifactId>protobuf-javanano</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf-lite</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf-nano</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-utils</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-metrics</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-instance</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-worker</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>net.jodah</groupId>
- <artifactId>typetools</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-api</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.glassfish.jersey.media</groupId>
- <artifactId>jersey-media-multipart</artifactId>
</dependency>
<!-- functions related dependencies (end) -->
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
index bb2228c..553f1c6 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
@@ -30,9 +30,9 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonParser;
import com.google.gson.reflect.TypeToken;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufUtil;
-import io.netty.buffer.Unpooled;
+import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBuf;
+import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBufUtil;
+import org.apache.pulsar.functions.shaded.io.netty.buffer.Unpooled;
import java.net.MalformedURLException;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml
index ee246f3..68dd322 100644
--- a/pulsar-common/pom.xml
+++ b/pulsar-common/pom.xml
@@ -70,12 +70,21 @@
<groupId>org.apache.bookkeeper</groupId>
<artifactId>circe-checksum</artifactId>
<version>${bookkeeper.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-tcnative-boringssl-static</artifactId>
- <version>2.0.7.Final</version>
</dependency>
</dependencies>
</project>
diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml
index 829f080..1a0405b 100644
--- a/pulsar-functions/pom.xml
+++ b/pulsar-functions/pom.xml
@@ -38,6 +38,8 @@
<module>metrics</module>
<module>instance</module>
<module>runtime</module>
+ <module>runtime-shaded</module>
+ <module>runtime-all</module>
<module>worker</module>
<module>worker-shaded</module>
<module>java-examples</module>
diff --git a/pulsar-client-tools/pom.xml b/pulsar-functions/runtime-all/pom.xml
similarity index 53%
copy from pulsar-client-tools/pom.xml
copy to pulsar-functions/runtime-all/pom.xml
index 693eca1..083a1a3 100644
--- a/pulsar-client-tools/pom.xml
+++ b/pulsar-functions/runtime-all/pom.xml
@@ -1,3 +1,4 @@
+<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
@@ -19,65 +20,24 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
+
<parent>
<groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar</artifactId>
+ <artifactId>pulsar-functions</artifactId>
<version>2.0.0-incubating-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
- <artifactId>pulsar-client-tools</artifactId>
- <name>Pulsar Client Tools</name>
- <description>Pulsar Client Tools</description>
+ <artifactId>pulsar-functions-runtime-all</artifactId>
+ <name>Pulsar Functions :: Runtime All</name>
<dependencies>
<dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-client-admin-original</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-client-original</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.asynchttpclient</groupId>
- <artifactId>async-http-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
-
- <!-- functions related dependencies (begin) -->
-
- <dependency>
<groupId>${project.groupId}</groupId>
- <artifactId>pulsar-functions-worker-shaded</artifactId>
- <version>${project.version}</version>
+ <artifactId>pulsar-functions-runtime-shaded</artifactId>
+ <version>${project.parent.version}</version>
<exclusions>
<exclusion>
<groupId>io.grpc</groupId>
@@ -113,6 +73,10 @@
</exclusion>
<exclusion>
<groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-functions-utils</artifactId>
</exclusion>
<exclusion>
@@ -127,35 +91,79 @@
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-functions-instance</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-worker</artifactId>
- </exclusion>
</exclusions>
</dependency>
+ <!-- logging -->
<dependency>
- <groupId>net.jodah</groupId>
- <artifactId>typetools</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
</dependency>
-
<dependency>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-api</artifactId>
- <version>${project.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
</dependency>
-
<dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
</dependency>
<dependency>
- <groupId>org.glassfish.jersey.media</groupId>
- <artifactId>jersey-media-multipart</artifactId>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>pulsar-client-original</artifactId>
+ <version>${project.parent.version}</version>
</dependency>
-
- <!-- functions related dependencies (end) -->
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <finalName>java-instance</finalName>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.openjdk.jmh.Main</mainClass>
+ </transformer>
+ </transformers>
+ <artifactSet>
+ <excludes>
+ <exclude>io.netty:netty-common</exclude>
+ <exclude>io.netty:netty-buffer</exclude>
+ <exclude>io.netty:netty-codec-http2</exclude>
+ <exclude>io.netty:netty-codec-http</exclude>
+ <exclude>io.netty:netty-codec-socks</exclude>
+ <exclude>io.netty:netty-codec</exclude>
+ <exclude>io.netty:netty-handler</exclude>
+ <exclude>io.netty:netty-handler-proxy</exclude>
+ <exclude>io.netty:netty-transport</exclude>
+ <exclude>io.netty:netty-resolver</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <!-- Shading signed JARs will fail without
+ this. http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar -->
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/pulsar-functions/worker-shaded/pom.xml b/pulsar-functions/runtime-shaded/pom.xml
similarity index 55%
copy from pulsar-functions/worker-shaded/pom.xml
copy to pulsar-functions/runtime-shaded/pom.xml
index 7243431..e356940 100644
--- a/pulsar-functions/worker-shaded/pom.xml
+++ b/pulsar-functions/runtime-shaded/pom.xml
@@ -30,33 +30,74 @@
<relativePath>..</relativePath>
</parent>
- <artifactId>pulsar-functions-worker-shaded</artifactId>
- <name>Pulsar Functions :: Worker Shaded</name>
+ <artifactId>pulsar-functions-runtime-shaded</artifactId>
+ <name>Pulsar Functions :: Runtime Shaded</name>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
- <artifactId>pulsar-functions-worker</artifactId>
+ <artifactId>pulsar-functions-runtime</artifactId>
<version>${project.parent.version}</version>
<exclusions>
- <!-- exclude the dependencies already exists in bookkeeper-server-shaded -->
+ <!-- exclude `pulsar-client-admin-shaded-for-functions` here, this allows worker and runtime to use unshaded clients -->
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-client-admin-shaded-for-functions</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http2</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler-proxy</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.apache.bookkeeper</groupId>
<artifactId>bookkeeper-common</artifactId>
</exclusion>
- <!-- exclude `pulsar-client-admin-shaded-for-functions` here, this allows worker-runner and broker to use unshaded clients -->
<exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-client-admin-shaded-for-functions</artifactId>
+ <groupId>org.apache.bookkeeper.stats</groupId>
+ <artifactId>bookkeeper-stats-api</artifactId>
</exclusion>
</exclusions>
</dependency>
- <!-- bookkeeper-server-shaded include circe-checksum, bookkeeper-common and bookkeeper-stats-api -->
<dependency>
<groupId>org.apache.bookkeeper</groupId>
<artifactId>bookkeeper-server-shaded</artifactId>
</dependency>
+
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>4.1.12.Final</version>
+ </dependency>
+
</dependencies>
<build>
@@ -76,7 +117,6 @@
<minimizeJar>false</minimizeJar>
<artifactSet>
- <!-- package the dependencies that use protobuf & grpc and relocate protobuf -->
<includes>
<include>com.google.protobuf:protobuf-lite</include>
<include>com.google.protobuf:protobuf-java</include>
@@ -84,17 +124,42 @@
<include>com.google.protobuf:protobuf-java-util</include>
<include>com.google.instrumentation:instrumentation-api</include>
<include>com.google.api.grpc:proto-google-common-protos</include>
+ <include>com.google.auth:google-auth-library-credentials</include>
+ <include>com.google.errorprone:error_prone_annotations</include>
+ <include>com.squareup.okhttp:okhttp</include>
+ <include>com.squareup.okio:okio</include>
+ <include>org.inferred:freebuilder</include>
+ <include>com.google.googlejavaformat:google-java-format</include>
+ <include>com.google.errorprone:javac</include>
+ <include>net.jodah:typetools</include>
+ <include>com.beust:jcommander</include>
+ <include>com.fasterxml.jackson.dataformat:jackson-dataformat-yaml</include>
+ <include>org.yaml:snakeyaml</include>
<!-- dependencies use protobuf -->
<include>org.apache.pulsar:pulsar-functions-proto</include>
<include>org.apache.pulsar:pulsar-functions-utils</include>
<include>org.apache.pulsar:pulsar-functions-metrics</include>
<include>org.apache.pulsar:pulsar-functions-instance</include>
<include>org.apache.pulsar:pulsar-functions-runtime</include>
- <include>org.apache.pulsar:pulsar-functions-worker</include>
+ <include>org.apache.pulsar:pulsar-functions-api</include>
<!-- protobuf dependencies in grpc -->
<include>io.grpc:*</include>
<!-- bookkeeper key/value service -->
<include>org.apache.bookkeeper:stream-storage-java-client</include>
+ <include>io.netty:netty-all</include>
+
+ <!-- dependencies to include when using shaded pulsar client-->
+ <!--
+ <include>com.fasterxml.jackson.core:jackson-core</include>
+ <include>com.fasterxml.jackson.core:jackson-databind</include>
+ <include>com.fasterxml.jackson.core:jackson-annotations</include>
+ <include>org.apache.commons:commons-lang3</include>
+ <include>com.yahoo.datasketches:sketches-core</include>
+ <include>com.google.guava:guava</include>
+ <include>org.apache.bookkeeper:bookkeeper-common</include>
+ <include>org.apache.bookkeeper.stats:bookkeeper-stats-api</include>
+ <include>com.google.code.gson:gson</include>
+ -->
</includes>
</artifactSet>
<relocations>
@@ -103,6 +168,10 @@
<pattern>com.google.protobuf</pattern>
<shadedPattern>org.apache.pulsar.functions.shaded.com.google.protobuf</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>io.netty</pattern>
+ <shadedPattern>org.apache.pulsar.functions.shaded.io.netty</shadedPattern>
+ </relocation>
</relocations>
</configuration>
</execution>
diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml
index 88828dc..b671182 100644
--- a/pulsar-functions/runtime/pom.xml
+++ b/pulsar-functions/runtime/pom.xml
@@ -54,40 +54,6 @@
<build>
<plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.2</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <finalName>java-instance</finalName>
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.openjdk.jmh.Main</mainClass>
- </transformer>
- </transformers>
- <filters>
- <filter>
- <!-- Shading signed JARs will fail without
- this. http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar -->
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
- </plugin>
<!-- this task is used for copy all python files together to provide a instance directory
for running python instance -->
<plugin>
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
index d1ec87a..3460c04 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
@@ -26,6 +26,7 @@ import com.google.gson.Gson;
import com.google.protobuf.Empty;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
+import java.util.concurrent.ExecutionException;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.functions.instance.InstanceConfig;
@@ -39,6 +40,7 @@ import java.io.InputStream;
import java.net.ServerSocket;
import java.util.*;
import java.util.concurrent.CompletableFuture;
+import org.apache.pulsar.functions.proto.InstanceControlGrpc.InstanceControlFutureStub;
/**
* A function container implemented using java thread.
@@ -300,4 +302,31 @@ class ProcessRuntime implements Runtime {
}
return startupException;
}
+
+ public static void main(String[] args) throws ExecutionException, InterruptedException {
+ int port = Integer.parseInt(args[0]);
+
+ ManagedChannel channel = ManagedChannelBuilder.forAddress("127.0.0.1", port)
+ .usePlaintext(true)
+ .build();
+ InstanceControlFutureStub stub = InstanceControlGrpc.newFutureStub(channel);
+ ListenableFuture<FunctionStatus> response = stub.getFunctionStatus(Empty.newBuilder().build());
+ CompletableFuture<FunctionStatus> future = new CompletableFuture<>();
+ Futures.addCallback(response, new FutureCallback<FunctionStatus>() {
+ @Override
+ public void onFailure(Throwable throwable) {
+ log.info("GetFunctionStatus:", throwable);
+ future.completeExceptionally(throwable);
+ }
+
+ @Override
+ public void onSuccess(InstanceCommunication.FunctionStatus t) {
+ log.info("GetFunctionStatus: {}", t);
+ future.complete(t);
+ }
+ });
+ FunctionStatus status = future.get();
+
+ log.info("Function Status : {}", status);
+ }
}
diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml
index 242a3fc..4e4413c 100644
--- a/pulsar-functions/utils/pom.xml
+++ b/pulsar-functions/utils/pom.xml
@@ -40,6 +40,14 @@
<!-- exclude shaded dependencies to avoid conflicts -->
<exclusions>
<exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-client-admin-original</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-client-original</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</exclusion>
@@ -111,6 +119,12 @@
</dependency>
<dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>4.1.12.Final</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-functions-proto</artifactId>
<version>${project.version}</version>
diff --git a/pulsar-functions/worker-shaded/pom.xml b/pulsar-functions/worker-shaded/pom.xml
index 7243431..06f8f63 100644
--- a/pulsar-functions/worker-shaded/pom.xml
+++ b/pulsar-functions/worker-shaded/pom.xml
@@ -49,6 +49,34 @@
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-client-admin-shaded-for-functions</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http2</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler-proxy</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -56,6 +84,12 @@
<dependency>
<groupId>org.apache.bookkeeper</groupId>
<artifactId>bookkeeper-server-shaded</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml
index 6f408e7..3dab594 100644
--- a/pulsar-functions/worker/pom.xml
+++ b/pulsar-functions/worker/pom.xml
@@ -36,7 +36,85 @@
<dependency>
<groupId>${project.groupId}</groupId>
- <artifactId>pulsar-functions-runtime</artifactId>
+ <artifactId>pulsar-functions-runtime-shaded</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-lite</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf.nano</groupId>
+ <artifactId>protobuf-javanano</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf-lite</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf-nano</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-utils</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-metrics</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-instance</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-resolver</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-socks</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>pulsar-client-original</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>pulsar-client-admin-original</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionAssignmentTailer.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionAssignmentTailer.java
index 66a8d6d..0a4cdeb 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionAssignmentTailer.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionAssignmentTailer.java
@@ -18,7 +18,6 @@
*/
package org.apache.pulsar.functions.worker;
-import com.google.protobuf.InvalidProtocolBufferException;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.PulsarClientException;
@@ -78,7 +77,7 @@ public class FunctionAssignmentTailer
Request.AssignmentsUpdate assignmentsUpdate;
try {
assignmentsUpdate = Request.AssignmentsUpdate.parseFrom(msg.getData());
- } catch (InvalidProtocolBufferException e) {
+ } catch (IOException e) {
log.error("Received bad assignment update at message {}", msg.getMessageId(), e);
// TODO: find a better way to handle bad request
throw new RuntimeException(e);
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataTopicTailer.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataTopicTailer.java
index 29a890c..a373158 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataTopicTailer.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataTopicTailer.java
@@ -21,7 +21,6 @@ package org.apache.pulsar.functions.worker;
import java.io.IOException;
import java.util.function.Function;
-import com.google.protobuf.InvalidProtocolBufferException;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.PulsarClientException;
@@ -68,7 +67,7 @@ public class FunctionMetaDataTopicTailer
try {
serviceRequest = ServiceRequest.parseFrom(msg.getData());
- } catch (InvalidProtocolBufferException e) {
+ } catch (IOException e) {
log.error("Received bad service request at message {}", msg.getMessageId(), e);
// TODO: find a better way to handle bad request
throw new RuntimeException(e);
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/SchedulerManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/SchedulerManagerTest.java
index 54a245f..a5ada20 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/SchedulerManagerTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/SchedulerManagerTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.pulsar.functions.worker;
-import com.google.protobuf.InvalidProtocolBufferException;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.Producer;
@@ -94,8 +93,7 @@ public class SchedulerManagerTest {
}
@Test
- public void testSchedule() throws PulsarClientException, NoSuchMethodException, InterruptedException,
- TimeoutException, ExecutionException {
+ public void testSchedule() throws Exception {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
@@ -139,8 +137,7 @@ public class SchedulerManagerTest {
}
@Test
- public void testNothingNewToSchedule() throws InterruptedException, ExecutionException, NoSuchMethodException,
- InvalidProtocolBufferException, TimeoutException {
+ public void testNothingNewToSchedule() throws Exception {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
@@ -191,8 +188,7 @@ public class SchedulerManagerTest {
}
@Test
- public void testAddingFunctions() throws NoSuchMethodException, InterruptedException,
- InvalidProtocolBufferException, TimeoutException, ExecutionException {
+ public void testAddingFunctions() throws Exception {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
@@ -255,8 +251,7 @@ public class SchedulerManagerTest {
}
@Test
- public void testDeletingFunctions() throws NoSuchMethodException, InterruptedException,
- InvalidProtocolBufferException, TimeoutException, ExecutionException {
+ public void testDeletingFunctions() throws Exception {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
@@ -322,8 +317,7 @@ public class SchedulerManagerTest {
}
@Test
- public void testScalingUp() throws NoSuchMethodException, InterruptedException, InvalidProtocolBufferException,
- PulsarClientException, TimeoutException, ExecutionException {
+ public void testScalingUp() throws Exception {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
@@ -432,8 +426,7 @@ public class SchedulerManagerTest {
}
@Test
- public void testScalingDown() throws PulsarClientException, NoSuchMethodException, InterruptedException,
- InvalidProtocolBufferException, TimeoutException, ExecutionException {
+ public void testScalingDown() throws Exception {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
@@ -543,8 +536,7 @@ public class SchedulerManagerTest {
}
@Test
- public void testUpdate() throws PulsarClientException, NoSuchMethodException, InterruptedException,
- InvalidProtocolBufferException, TimeoutException, ExecutionException {
+ public void testUpdate() throws Exception {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
--
To stop receiving notification emails like this one, please contact
sijie@apache.org.