You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by GitBox <gi...@apache.org> on 2018/03/07 05:56:22 UTC
[GitHub] sijie closed pull request #1351: Shade pulsar-functions-runtime instead of pulsar-functions-worker
sijie closed pull request #1351: Shade pulsar-functions-runtime instead of pulsar-functions-worker
URL: https://github.com/apache/incubator-pulsar/pull/1351
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/all/pom.xml b/all/pom.xml
index 6273f4bf4..736870054 100644
--- a/all/pom.xml
+++ b/all/pom.xml
@@ -113,6 +113,15 @@
<!-- make sure the api examples are compiled before assembly -->
<scope>provided</scope>
</dependency>
+
+ <!-- runtime-all -->
+ <dependency>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-runtime-all</artifactId>
+ <version>${project.version}</version>
+ <!-- make sure the api examples are compiled before assembly -->
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/all/src/assemble/bin.xml b/all/src/assemble/bin.xml
index 3a994acf9..e74b68405 100644
--- a/all/src/assemble/bin.xml
+++ b/all/src/assemble/bin.xml
@@ -73,7 +73,7 @@
<fileMode>644</fileMode>
</file>
<file>
- <source>${basedir}/../pulsar-functions/runtime/target/java-instance.jar</source>
+ <source>${basedir}/../pulsar-functions/runtime-all/target/java-instance.jar</source>
<destName>java-instance.jar</destName>
<outputDirectory>instances</outputDirectory>
</file>
diff --git a/bin/pulsar b/bin/pulsar
index cafc3591b..0bd2228ae 100755
--- a/bin/pulsar
+++ b/bin/pulsar
@@ -78,7 +78,7 @@ fi
# find the java instance location
if [ ! -f "${JAVA_INSTANCE_JAR}" ]; then
# didn't find a released jar, then search the built jar
- BUILT_JAVA_INSTANCE_JAR="${FUNCTIONS_HOME}/runtime/target/java-instance.jar"
+ BUILT_JAVA_INSTANCE_JAR="${FUNCTIONS_HOME}/runtime-all/target/java-instance.jar"
if [ -z "${BUILT_JAVA_INSTANCE_JAR}" ]; then
echo "\nCouldn't find pulsar-functions java instance jar.";
echo "Make sure you've run 'mvn package'\n";
diff --git a/bin/pulsar-admin b/bin/pulsar-admin
index 2e4922562..c8b41da45 100755
--- a/bin/pulsar-admin
+++ b/bin/pulsar-admin
@@ -117,7 +117,7 @@ OPTS="$OPTS -Dpulsar.routing.appender.default=$PULSAR_ROUTING_APPENDER_DEFAULT"
# find the java instance location
if [ ! -f "${JAVA_INSTANCE_JAR}" ]; then
# didn't find a released jar, then search the built jar
- BUILT_JAVA_INSTANCE_JAR="${FUNCTIONS_HOME}/runtime/target/java-instance.jar"
+ BUILT_JAVA_INSTANCE_JAR="${FUNCTIONS_HOME}/runtime-all/target/java-instance.jar"
if [ -f "${BUILT_JAVA_INSTANCE_JAR}" ]; then
JAVA_INSTANCE_JAR=${BUILT_JAVA_INSTANCE_JAR}
else
diff --git a/pom.xml b/pom.xml
index 78c6641ee..081fcdb32 100644
--- a/pom.xml
+++ b/pom.xml
@@ -318,6 +318,12 @@ flexible messaging model and an intuitive client API.</description>
<version>${netty.version}</version>
</dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ <version>2.0.7.Final</version>
+ </dependency>
+
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
@@ -731,9 +737,8 @@ flexible messaging model and an intuitive client API.</description>
<dependency>
<!-- We use MockedBookKeeper in many unit tests -->
<groupId>org.apache.bookkeeper</groupId>
- <artifactId>bookkeeper-server</artifactId>
+ <artifactId>bookkeeper-server-tests-shaded</artifactId>
<version>${bookkeeper.version}</version>
- <classifier>tests</classifier>
<scope>test</scope>
<exclusions>
<exclusion>
diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml
index abbc8313f..abead555d 100644
--- a/pulsar-broker/pom.xml
+++ b/pulsar-broker/pom.xml
@@ -54,6 +54,18 @@
<artifactId>netty-all</artifactId>
</dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>pulsar-common</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.bookkeeper</groupId>
+ <artifactId>circe-checksum</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>pulsar-client-original</artifactId>
@@ -123,88 +135,8 @@
<dependency>
<groupId>${project.groupId}</groupId>
- <artifactId>pulsar-functions-worker-shaded</artifactId>
+ <artifactId>pulsar-functions-worker</artifactId>
<version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-lite</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf.nano</groupId>
- <artifactId>protobuf-javanano</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf-lite</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf-nano</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-utils</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-metrics</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-instance</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-worker</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>net.jodah</groupId>
- <artifactId>typetools</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-api</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.glassfish.jersey.media</groupId>
- <artifactId>jersey-media-multipart</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.distributedlog</groupId>
- <artifactId>distributedlog-core-shaded</artifactId>
</dependency>
<!-- functions related dependencies (end) -->
diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java
index e57aa8444..2cd664a10 100644
--- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java
+++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java
@@ -31,8 +31,8 @@
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNull;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufUtil;
+import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBuf;
+import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBufUtil;
import java.io.File;
import java.net.URI;
import java.util.concurrent.atomic.AtomicReference;
diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml
index 693eca1e2..5779b7846 100644
--- a/pulsar-client-tools/pom.xml
+++ b/pulsar-client-tools/pom.xml
@@ -76,83 +76,8 @@
<dependency>
<groupId>${project.groupId}</groupId>
- <artifactId>pulsar-functions-worker-shaded</artifactId>
+ <artifactId>pulsar-functions-worker</artifactId>
<version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-lite</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf.nano</groupId>
- <artifactId>protobuf-javanano</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf-lite</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf-nano</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-utils</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-metrics</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-instance</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-worker</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>net.jodah</groupId>
- <artifactId>typetools</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-functions-api</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.glassfish.jersey.media</groupId>
- <artifactId>jersey-media-multipart</artifactId>
</dependency>
<!-- functions related dependencies (end) -->
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
index bb2228c57..553f1c658 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
@@ -30,9 +30,9 @@
import com.google.gson.GsonBuilder;
import com.google.gson.JsonParser;
import com.google.gson.reflect.TypeToken;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufUtil;
-import io.netty.buffer.Unpooled;
+import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBuf;
+import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBufUtil;
+import org.apache.pulsar.functions.shaded.io.netty.buffer.Unpooled;
import java.net.MalformedURLException;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml
index ee246f3d1..68dd3222d 100644
--- a/pulsar-common/pom.xml
+++ b/pulsar-common/pom.xml
@@ -70,12 +70,21 @@
<groupId>org.apache.bookkeeper</groupId>
<artifactId>circe-checksum</artifactId>
<version>${bookkeeper.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-tcnative-boringssl-static</artifactId>
- <version>2.0.7.Final</version>
</dependency>
</dependencies>
</project>
diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml
index 829f080ee..1a0405bf9 100644
--- a/pulsar-functions/pom.xml
+++ b/pulsar-functions/pom.xml
@@ -38,6 +38,8 @@
<module>metrics</module>
<module>instance</module>
<module>runtime</module>
+ <module>runtime-shaded</module>
+ <module>runtime-all</module>
<module>worker</module>
<module>worker-shaded</module>
<module>java-examples</module>
diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml
new file mode 100644
index 000000000..083a1a381
--- /dev/null
+++ b/pulsar-functions/runtime-all/pom.xml
@@ -0,0 +1,169 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions</artifactId>
+ <version>2.0.0-incubating-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>pulsar-functions-runtime-all</artifactId>
+ <name>Pulsar Functions :: Runtime All</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>pulsar-functions-runtime-shaded</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-lite</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf.nano</groupId>
+ <artifactId>protobuf-javanano</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf-lite</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf-nano</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-utils</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-metrics</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-instance</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- logging -->
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>pulsar-client-original</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <finalName>java-instance</finalName>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.openjdk.jmh.Main</mainClass>
+ </transformer>
+ </transformers>
+ <artifactSet>
+ <excludes>
+ <exclude>io.netty:netty-common</exclude>
+ <exclude>io.netty:netty-buffer</exclude>
+ <exclude>io.netty:netty-codec-http2</exclude>
+ <exclude>io.netty:netty-codec-http</exclude>
+ <exclude>io.netty:netty-codec-socks</exclude>
+ <exclude>io.netty:netty-codec</exclude>
+ <exclude>io.netty:netty-handler</exclude>
+ <exclude>io.netty:netty-handler-proxy</exclude>
+ <exclude>io.netty:netty-transport</exclude>
+ <exclude>io.netty:netty-resolver</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <!-- Shading signed JARs will fail without
+ this. http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar -->
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/pulsar-functions/runtime-shaded/pom.xml b/pulsar-functions/runtime-shaded/pom.xml
new file mode 100644
index 000000000..e356940e0
--- /dev/null
+++ b/pulsar-functions/runtime-shaded/pom.xml
@@ -0,0 +1,182 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions</artifactId>
+ <version>2.0.0-incubating-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>pulsar-functions-runtime-shaded</artifactId>
+ <name>Pulsar Functions :: Runtime Shaded</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>pulsar-functions-runtime</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <!-- exclude `pulsar-client-admin-shaded-for-functions` here, this allows worker and runtime to use unshaded clients -->
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-client-admin-shaded-for-functions</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http2</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler-proxy</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.bookkeeper</groupId>
+ <artifactId>bookkeeper-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.bookkeeper.stats</groupId>
+ <artifactId>bookkeeper-stats-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.bookkeeper</groupId>
+ <artifactId>bookkeeper-server-shaded</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>4.1.12.Final</version>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+ <minimizeJar>false</minimizeJar>
+
+ <artifactSet>
+ <includes>
+ <include>com.google.protobuf:protobuf-lite</include>
+ <include>com.google.protobuf:protobuf-java</include>
+ <include>com.google.protobuf.nano:protobuf-javanano</include>
+ <include>com.google.protobuf:protobuf-java-util</include>
+ <include>com.google.instrumentation:instrumentation-api</include>
+ <include>com.google.api.grpc:proto-google-common-protos</include>
+ <include>com.google.auth:google-auth-library-credentials</include>
+ <include>com.google.errorprone:error_prone_annotations</include>
+ <include>com.squareup.okhttp:okhttp</include>
+ <include>com.squareup.okio:okio</include>
+ <include>org.inferred:freebuilder</include>
+ <include>com.google.googlejavaformat:google-java-format</include>
+ <include>com.google.errorprone:javac</include>
+ <include>net.jodah:typetools</include>
+ <include>com.beust:jcommander</include>
+ <include>com.fasterxml.jackson.dataformat:jackson-dataformat-yaml</include>
+ <include>org.yaml:snakeyaml</include>
+ <!-- dependencies use protobuf -->
+ <include>org.apache.pulsar:pulsar-functions-proto</include>
+ <include>org.apache.pulsar:pulsar-functions-utils</include>
+ <include>org.apache.pulsar:pulsar-functions-metrics</include>
+ <include>org.apache.pulsar:pulsar-functions-instance</include>
+ <include>org.apache.pulsar:pulsar-functions-runtime</include>
+ <include>org.apache.pulsar:pulsar-functions-api</include>
+ <!-- protobuf dependencies in grpc -->
+ <include>io.grpc:*</include>
+ <!-- bookkeeper key/value service -->
+ <include>org.apache.bookkeeper:stream-storage-java-client</include>
+ <include>io.netty:netty-all</include>
+
+ <!-- dependencies to include when using shaded pulsar client-->
+ <!--
+ <include>com.fasterxml.jackson.core:jackson-core</include>
+ <include>com.fasterxml.jackson.core:jackson-databind</include>
+ <include>com.fasterxml.jackson.core:jackson-annotations</include>
+ <include>org.apache.commons:commons-lang3</include>
+ <include>com.yahoo.datasketches:sketches-core</include>
+ <include>com.google.guava:guava</include>
+ <include>org.apache.bookkeeper:bookkeeper-common</include>
+ <include>org.apache.bookkeeper.stats:bookkeeper-stats-api</include>
+ <include>com.google.code.gson:gson</include>
+ -->
+ </includes>
+ </artifactSet>
+ <relocations>
+ <!-- bookkeeper shading rule -->
+ <relocation>
+ <pattern>com.google.protobuf</pattern>
+ <shadedPattern>org.apache.pulsar.functions.shaded.com.google.protobuf</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>io.netty</pattern>
+ <shadedPattern>org.apache.pulsar.functions.shaded.io.netty</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml
index 88828dc35..b67118252 100644
--- a/pulsar-functions/runtime/pom.xml
+++ b/pulsar-functions/runtime/pom.xml
@@ -54,40 +54,6 @@
<build>
<plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.2</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <finalName>java-instance</finalName>
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.openjdk.jmh.Main</mainClass>
- </transformer>
- </transformers>
- <filters>
- <filter>
- <!-- Shading signed JARs will fail without
- this. http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar -->
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
- </plugin>
<!-- this task is used for copy all python files together to provide a instance directory
for running python instance -->
<plugin>
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
index d1ec87aa7..3460c0453 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
@@ -26,6 +26,7 @@
import com.google.protobuf.Empty;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
+import java.util.concurrent.ExecutionException;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.functions.instance.InstanceConfig;
@@ -39,6 +40,7 @@
import java.net.ServerSocket;
import java.util.*;
import java.util.concurrent.CompletableFuture;
+import org.apache.pulsar.functions.proto.InstanceControlGrpc.InstanceControlFutureStub;
/**
* A function container implemented using java thread.
@@ -300,4 +302,31 @@ public Exception getDeathException() {
}
return startupException;
}
+
+ public static void main(String[] args) throws ExecutionException, InterruptedException {
+ int port = Integer.parseInt(args[0]);
+
+ ManagedChannel channel = ManagedChannelBuilder.forAddress("127.0.0.1", port)
+ .usePlaintext(true)
+ .build();
+ InstanceControlFutureStub stub = InstanceControlGrpc.newFutureStub(channel);
+ ListenableFuture<FunctionStatus> response = stub.getFunctionStatus(Empty.newBuilder().build());
+ CompletableFuture<FunctionStatus> future = new CompletableFuture<>();
+ Futures.addCallback(response, new FutureCallback<FunctionStatus>() {
+ @Override
+ public void onFailure(Throwable throwable) {
+ log.info("GetFunctionStatus:", throwable);
+ future.completeExceptionally(throwable);
+ }
+
+ @Override
+ public void onSuccess(InstanceCommunication.FunctionStatus t) {
+ log.info("GetFunctionStatus: {}", t);
+ future.complete(t);
+ }
+ });
+ FunctionStatus status = future.get();
+
+ log.info("Function Status : {}", status);
+ }
}
diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml
index 242a3fc51..4e4413cd4 100644
--- a/pulsar-functions/utils/pom.xml
+++ b/pulsar-functions/utils/pom.xml
@@ -39,6 +39,14 @@
<version>${project.version}</version>
<!-- exclude shaded dependencies to avoid conflicts -->
<exclusions>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-client-admin-original</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-client-original</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
@@ -110,6 +118,12 @@
</exclusions>
</dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>4.1.12.Final</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-functions-proto</artifactId>
diff --git a/pulsar-functions/worker-shaded/pom.xml b/pulsar-functions/worker-shaded/pom.xml
index 724343125..06f8f6326 100644
--- a/pulsar-functions/worker-shaded/pom.xml
+++ b/pulsar-functions/worker-shaded/pom.xml
@@ -49,6 +49,34 @@
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-client-admin-shaded-for-functions</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http2</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler-proxy</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -56,6 +84,12 @@
<dependency>
<groupId>org.apache.bookkeeper</groupId>
<artifactId>bookkeeper-server-shaded</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml
index 6f408e716..3dab594f6 100644
--- a/pulsar-functions/worker/pom.xml
+++ b/pulsar-functions/worker/pom.xml
@@ -36,7 +36,85 @@
<dependency>
<groupId>${project.groupId}</groupId>
- <artifactId>pulsar-functions-runtime</artifactId>
+ <artifactId>pulsar-functions-runtime-shaded</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-lite</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf.nano</groupId>
+ <artifactId>protobuf-javanano</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf-lite</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf-nano</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-utils</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-metrics</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-instance</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-resolver</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-socks</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>pulsar-client-original</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>pulsar-client-admin-original</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionAssignmentTailer.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionAssignmentTailer.java
index 66a8d6d92..0a4cdeb0b 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionAssignmentTailer.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionAssignmentTailer.java
@@ -18,7 +18,6 @@
*/
package org.apache.pulsar.functions.worker;
-import com.google.protobuf.InvalidProtocolBufferException;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.PulsarClientException;
@@ -78,7 +77,7 @@ public void accept(Message msg) {
Request.AssignmentsUpdate assignmentsUpdate;
try {
assignmentsUpdate = Request.AssignmentsUpdate.parseFrom(msg.getData());
- } catch (InvalidProtocolBufferException e) {
+ } catch (IOException e) {
log.error("Received bad assignment update at message {}", msg.getMessageId(), e);
// TODO: find a better way to handle bad request
throw new RuntimeException(e);
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataTopicTailer.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataTopicTailer.java
index 29a890c40..a37315899 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataTopicTailer.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataTopicTailer.java
@@ -21,7 +21,6 @@
import java.io.IOException;
import java.util.function.Function;
-import com.google.protobuf.InvalidProtocolBufferException;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.PulsarClientException;
@@ -68,7 +67,7 @@ public void processRequest(Message msg) {
try {
serviceRequest = ServiceRequest.parseFrom(msg.getData());
- } catch (InvalidProtocolBufferException e) {
+ } catch (IOException e) {
log.error("Received bad service request at message {}", msg.getMessageId(), e);
// TODO: find a better way to handle bad request
throw new RuntimeException(e);
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/SchedulerManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/SchedulerManagerTest.java
index 54a245f72..a5ada20e8 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/SchedulerManagerTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/SchedulerManagerTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.pulsar.functions.worker;
-import com.google.protobuf.InvalidProtocolBufferException;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.Producer;
@@ -94,8 +93,7 @@ public void setup() throws PulsarClientException {
}
@Test
- public void testSchedule() throws PulsarClientException, NoSuchMethodException, InterruptedException,
- TimeoutException, ExecutionException {
+ public void testSchedule() throws Exception {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
@@ -139,8 +137,7 @@ public void testSchedule() throws PulsarClientException, NoSuchMethodException,
}
@Test
- public void testNothingNewToSchedule() throws InterruptedException, ExecutionException, NoSuchMethodException,
- InvalidProtocolBufferException, TimeoutException {
+ public void testNothingNewToSchedule() throws Exception {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
@@ -191,8 +188,7 @@ public void testNothingNewToSchedule() throws InterruptedException, ExecutionExc
}
@Test
- public void testAddingFunctions() throws NoSuchMethodException, InterruptedException,
- InvalidProtocolBufferException, TimeoutException, ExecutionException {
+ public void testAddingFunctions() throws Exception {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
@@ -255,8 +251,7 @@ public void testAddingFunctions() throws NoSuchMethodException, InterruptedExcep
}
@Test
- public void testDeletingFunctions() throws NoSuchMethodException, InterruptedException,
- InvalidProtocolBufferException, TimeoutException, ExecutionException {
+ public void testDeletingFunctions() throws Exception {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
@@ -322,8 +317,7 @@ public void testDeletingFunctions() throws NoSuchMethodException, InterruptedExc
}
@Test
- public void testScalingUp() throws NoSuchMethodException, InterruptedException, InvalidProtocolBufferException,
- PulsarClientException, TimeoutException, ExecutionException {
+ public void testScalingUp() throws Exception {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
@@ -432,8 +426,7 @@ public void testScalingUp() throws NoSuchMethodException, InterruptedException,
}
@Test
- public void testScalingDown() throws PulsarClientException, NoSuchMethodException, InterruptedException,
- InvalidProtocolBufferException, TimeoutException, ExecutionException {
+ public void testScalingDown() throws Exception {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
@@ -543,8 +536,7 @@ public void testScalingDown() throws PulsarClientException, NoSuchMethodExceptio
}
@Test
- public void testUpdate() throws PulsarClientException, NoSuchMethodException, InterruptedException,
- InvalidProtocolBufferException, TimeoutException, ExecutionException {
+ public void testUpdate() throws Exception {
List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
long version = 5;
Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder()
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services