You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2025/08/26 10:32:51 UTC
(pulsar) 06/07: [improve][io] Upgrade AWS SDK v1 & v2, Kinesis KPL and KPC versions (#24661)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit e52b4e0fa62b8afd2b6b9525f636d0b72f7986c0
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Tue Aug 26 07:44:05 2025 +0300
[improve][io] Upgrade AWS SDK v1 & v2, Kinesis KPL and KPC versions (#24661)
(cherry picked from commit dc2ab26675bdbe11eabdbf86ff01f928bb820ebe)
---
docker/kinesis-producer-alpine/Dockerfile | 2 +-
docker/kinesis-producer-alpine/README.md | 2 +-
docker/kinesis-producer-alpine/build-alpine.sh | 112 +++++++++-------
.../kinesis_producer_alpine.patch | 23 ++--
docker/pulsar-all/pom.xml | 2 +-
pom.xml | 11 +-
pulsar-io/aws/pom.xml | 1 -
pulsar-io/kinesis-kpl-shaded/pom.xml | 149 +++++++++++++++++++++
pulsar-io/kinesis/pom.xml | 24 ++--
.../org/apache/pulsar/io/kinesis/KinesisSink.java | 15 ++-
.../pulsar/io/kinesis/KinesisSinkConfig.java | 4 +-
.../pulsar/io/kinesis/KinesisSinkConfigTest.java | 2 +-
pulsar-io/pom.xml | 2 +
tests/integration/pom.xml | 2 +-
14 files changed, 271 insertions(+), 80 deletions(-)
diff --git a/docker/kinesis-producer-alpine/Dockerfile b/docker/kinesis-producer-alpine/Dockerfile
index ac04e69aea0..a33dc82f158 100644
--- a/docker/kinesis-producer-alpine/Dockerfile
+++ b/docker/kinesis-producer-alpine/Dockerfile
@@ -23,7 +23,7 @@ ARG ALPINE_VERSION=3.21
# Build stage
FROM alpine:$ALPINE_VERSION AS kinesis-producer-build
-ENV KINESIS_PRODUCER_LIB_VERSION=0.15.12
+ENV KINESIS_PRODUCER_LIB_VERSION=1.0.4
# Install build dependencies
RUN apk update && apk add --no-cache \
diff --git a/docker/kinesis-producer-alpine/README.md b/docker/kinesis-producer-alpine/README.md
index dc21b05dec3..e3fa0f88602 100644
--- a/docker/kinesis-producer-alpine/README.md
+++ b/docker/kinesis-producer-alpine/README.md
@@ -32,7 +32,7 @@ This image only needs to be re-created when we want to upgrade to a newer versio
2. Rebuild the image and push it to Docker Hub:
```
IMAGE=apachepulsar/pulsar-io-kinesis-sink-kinesis_producer
-KINESIS_PRODUCER_VERSION=0.15.12
+KINESIS_PRODUCER_VERSION=1.0.4
docker buildx build --platform=linux/amd64,linux/arm64 \
-t "$IMAGE:$KINESIS_PRODUCER_VERSION" -t "$IMAGE:${KINESIS_PRODUCER_VERSION}-$(date -I)" \
. --push
diff --git a/docker/kinesis-producer-alpine/build-alpine.sh b/docker/kinesis-producer-alpine/build-alpine.sh
index 6ea41c55582..52ae9c1a32e 100644
--- a/docker/kinesis-producer-alpine/build-alpine.sh
+++ b/docker/kinesis-producer-alpine/build-alpine.sh
@@ -20,13 +20,25 @@
set -e
set -x
+SILENT="n"
-INSTALL_DIR=/build/third_party
-AWS_SDK_CPP_VERSION="1.11.420"
-PROTOBUF_VERSION="3.11.4"
-BOOST_VERSION="1.76.0"
-BOOST_VERSION_UNDERSCORED="${BOOST_VERSION//\./_}"
+silence() {
+ if [ -n "$SILENT" ]; then
+ "$@" >/dev/null
+ else
+ "$@"
+ fi
+}
+
+BOOST_VERSION="1.88.0"
+BOOST_VERSION_UNDERSCORED="${BOOST_VERSION//\./_}" # convert from 1.76.0 to 1_76_0
+PROTOBUF_VERSION="21.12"
+AWS_SDK_CPP_VERSION="1.11.615"
+
+LIB_BOOST="https://archives.boost.io/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORED}.tar.gz"
+LIB_PROTOBUF="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz"
+INSTALL_DIR=/build/third_party
# Create install directory
mkdir -p $INSTALL_DIR
@@ -39,61 +51,67 @@ export LD_LIBRARY_PATH="$INSTALL_DIR/lib:$LD_LIBRARY_PATH"
cd $INSTALL_DIR
-# Build protobuf
-if [ ! -d "protobuf-${PROTOBUF_VERSION}" ]; then
- curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz
- tar xf protobuf-all-${PROTOBUF_VERSION}.tar.gz
- rm protobuf-all-${PROTOBUF_VERSION}.tar.gz
-
- cd protobuf-${PROTOBUF_VERSION}
- ./configure --prefix=${INSTALL_DIR} \
- --disable-shared \
- CFLAGS="-fPIC" \
- CXXFLAGS="-fPIC ${CXXFLAGS}" \
- --with-pic
- make -j4
- make install
- cd ..
-fi
-
-# Build Boost
+SED="sed -i"
+CMAKE=$(which cmake3 &>/dev/null && echo "cmake3 " || echo "cmake")
+function _curl {
+ curl -L "$@"
+}
+
+function conf {
+ silence ./configure \
+ --prefix="$INSTALL_DIR" \
+ LD_LIBRARY_PATH="$LD_LIBRARY_PATH" \
+ LDFLAGS="$LDFLAGS" \
+ CXXFLAGS="$CXXFLAGS" \
+ C_INCLUDE_PATH="$C_INCLUDE_PATH" \
+ "$@"
+}
+
+# Boost C++ Libraries
if [ ! -d "boost_${BOOST_VERSION_UNDERSCORED}" ]; then
- curl -LO https://archives.boost.io/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORED}.tar.gz
- tar xf boost_${BOOST_VERSION_UNDERSCORED}.tar.gz
- rm boost_${BOOST_VERSION_UNDERSCORED}.tar.gz
+ _curl "$LIB_BOOST" >boost.tgz
+ tar xf boost.tgz
+ rm boost.tgz
cd boost_${BOOST_VERSION_UNDERSCORED}
- BOOST_LIBS="regex,thread,log,system,random,filesystem,chrono,atomic,date_time,program_options,test"
+ LIBS="atomic,chrono,log,system,test,random,regex,thread,filesystem"
+ OPTS="--build-type=minimal --layout=system --prefix=$INSTALL_DIR link=static threading=multi release install"
- ./bootstrap.sh --with-libraries=$BOOST_LIBS --with-toolset=gcc
+ silence ./bootstrap.sh --with-libraries="$LIBS" --with-toolset=gcc
+ silence ./b2 toolset=gcc $OPTS
- ./b2 \
- -j4 \
- variant=release \
- link=static \
- threading=multi \
- runtime-link=static \
- --prefix=${INSTALL_DIR} \
- cxxflags="-fPIC ${CXXFLAGS}" \
- install
+ cd ..
+fi
+
+# Google Protocol Buffers
+if [ ! -d "protobuf-${PROTOBUF_VERSION}" ]; then
+ _curl "$LIB_PROTOBUF" >protobuf.tgz
+ tar xf protobuf.tgz
+ rm protobuf.tgz
+
+ cd protobuf-${PROTOBUF_VERSION}
+ silence conf --enable-shared=no
+ silence make -j 4
+ silence make install
cd ..
fi
-# Download and build AWS SDK
+# AWS C++ SDK
if [ ! -d "aws-sdk-cpp" ]; then
- git clone --depth 1 --branch ${AWS_SDK_CPP_VERSION} https://github.com/awslabs/aws-sdk-cpp.git aws-sdk-cpp
+ git clone https://github.com/awslabs/aws-sdk-cpp.git aws-sdk-cpp
pushd aws-sdk-cpp
- git config submodule.fetchJobs 8
- git submodule update --init --depth 1 --recursive
+ git checkout ${AWS_SDK_CPP_VERSION}
+ git submodule update --init --recursive
popd
rm -rf aws-sdk-cpp-build
mkdir aws-sdk-cpp-build
+
cd aws-sdk-cpp-build
- cmake \
+ silence $CMAKE \
-DBUILD_ONLY="kinesis;monitoring;sts" \
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DSTATIC_LINKING=1 \
@@ -105,15 +123,19 @@ if [ ! -d "aws-sdk-cpp" ]; then
-DCMAKE_FIND_FRAMEWORK=LAST \
-DENABLE_TESTING="OFF" \
../aws-sdk-cpp
- make -j4
- make install
+ silence make -j4
+ silence make install
+
cd ..
+
fi
+cd ..
+
# Build the native kinesis producer
cd /build/amazon-kinesis-producer
ln -fs ../third_party
-cmake -DCMAKE_PREFIX_PATH="$INSTALL_DIR" -DCMAKE_BUILD_TYPE=RelWithDebInfo .
+$CMAKE -DCMAKE_PREFIX_PATH="$INSTALL_DIR" -DCMAKE_BUILD_TYPE=RelWithDebInfo .
make -j4
FINAL_DIR=/opt/amazon-kinesis-producer
diff --git a/docker/kinesis-producer-alpine/kinesis_producer_alpine.patch b/docker/kinesis-producer-alpine/kinesis_producer_alpine.patch
index d1ddd6a8550..d31e61a3f8e 100644
--- a/docker/kinesis-producer-alpine/kinesis_producer_alpine.patch
+++ b/docker/kinesis-producer-alpine/kinesis_producer_alpine.patch
@@ -1,10 +1,9 @@
-From 96ba2eb7145363586529e6c770dcc0920bf04ac2 Mon Sep 17 00:00:00 2001
+From 76518befa2b3b79931044267722feecfe1465d19 Mon Sep 17 00:00:00 2001
From: Lari Hotari <lh...@users.noreply.github.com>
-Date: Wed, 18 Dec 2024 17:16:02 +0200
-Subject: [PATCH] Adapt build for Alpine, fix issue with NULL_BACKTRACE support
+Date: Fri, 22 Aug 2025 21:25:49 +0300
+Subject: [PATCH] [PATCH] Adapt build for Alpine, fix issue with NULL_BACKTRACE
+ support
-- also use dynamic linking to some libraries (zlib, openssl, libz, libcurl, libcrypto)
- to reduce binary size
---
CMakeLists.txt | 20 +++++++++++---------
aws/utils/backtrace/bsd_backtrace.cc | 2 +-
@@ -14,12 +13,12 @@ Subject: [PATCH] Adapt build for Alpine, fix issue with NULL_BACKTRACE support
5 files changed, 15 insertions(+), 16 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 2dd7084..2ba47e6 100644
+index 8803f5c..6152b3d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
-@@ -213,22 +213,24 @@ set(STATIC_LIBS
- boost_chrono)
+@@ -213,8 +213,10 @@ set(STATIC_LIBS
+ find_package(Protobuf REQUIRED)
find_package(Threads)
-find_package(ZLIB)
+find_package(ZLIB REQUIRED)
@@ -27,6 +26,12 @@ index 2dd7084..2ba47e6 100644
+find_package(OpenSSL REQUIRED)
+find_package(CURL REQUIRED)
+ # Specify the output directory for generated protobuf files
+ set(PROTO_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/aws/kinesis/protobuf/)
+@@ -239,19 +241,19 @@ foreach(proto ${PROTO_FILES})
+ )
+ endforeach()
+
-add_library(LibCrypto STATIC IMPORTED)
-set_property(TARGET LibCrypto PROPERTY IMPORTED_LOCATION ${THIRD_PARTY_LIB_DIR}/libcrypto.a)
+add_library(LibCrypto SHARED IMPORTED)
@@ -123,5 +128,5 @@ index b58ab0e..f483c77 100644
#include <cstring>
#include <cstdint>
--
-2.47.1
+2.51.0
diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml
index 4f6907881e0..bb8d4363352 100644
--- a/docker/pulsar-all/pom.xml
+++ b/docker/pulsar-all/pom.xml
@@ -82,7 +82,7 @@
<!-- include the docker image only when docker profile is active -->
<properties>
<!-- reference to the image built from docker/kinesis-producer-alpine/Dockerfile -->
- <PULSAR_IO_KINESIS_KPL_IMAGE>apachepulsar/pulsar-io-kinesis-sink-kinesis_producer:0.15.12</PULSAR_IO_KINESIS_KPL_IMAGE>
+ <PULSAR_IO_KINESIS_KPL_IMAGE>apachepulsar/pulsar-io-kinesis-sink-kinesis_producer:1.0.4</PULSAR_IO_KINESIS_KPL_IMAGE>
</properties>
<dependencies>
<dependency>
diff --git a/pom.xml b/pom.xml
index f8b31c61451..aa099c9ef09 100644
--- a/pom.xml
+++ b/pom.xml
@@ -227,7 +227,8 @@ flexible messaging model and an intuitive client API.</description>
<aerospike-client.version>4.5.0</aerospike-client.version>
<kafka-client.version>3.9.0</kafka-client.version>
<rabbitmq-client.version>5.18.0</rabbitmq-client.version>
- <aws-sdk.version>1.12.638</aws-sdk.version>
+ <aws-sdk.version>1.12.788</aws-sdk.version>
+ <aws-sdk2.version>2.32.28</aws-sdk2.version>
<avro.version>1.11.4</avro.version>
<joda.version>2.10.10</joda.version>
<jclouds.version>2.6.0</jclouds.version>
@@ -1362,6 +1363,14 @@ flexible messaging model and an intuitive client API.</description>
<scope>import</scope>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>bom</artifactId>
+ <version>${aws-sdk2.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.distributedlog</groupId>
<artifactId>distributedlog-core</artifactId>
diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml
index bca87245832..5ac6c61bc27 100644
--- a/pulsar-io/aws/pom.xml
+++ b/pulsar-io/aws/pom.xml
@@ -51,7 +51,6 @@
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>sts</artifactId>
- <version>2.10.56</version>
</dependency>
<!-- /aws dependencies -->
diff --git a/pulsar-io/kinesis-kpl-shaded/pom.xml b/pulsar-io/kinesis-kpl-shaded/pom.xml
new file mode 100644
index 00000000000..2574d6df23d
--- /dev/null
+++ b/pulsar-io/kinesis-kpl-shaded/pom.xml
@@ -0,0 +1,149 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar</artifactId>
+ <version>4.0.7-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>pulsar-io-kinesis-kpl-shaded</artifactId>
+ <name>Pulsar IO :: Kinesis KPL Shaded</name>
+
+ <!--
+ There's a need to shade the protobuf-java dependency in the AWS Kinesis Producer Library (KPL)
+ since it conflicts at runtime with Pulsar Functions GRPC protobuf-java version.
+ -->
+
+ <properties>
+ <amazon-kinesis-producer.version>1.0.4</amazon-kinesis-producer.version>
+ <protobuf-java.version>4.29.0</protobuf-java.version>
+ </properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <!-- enforce protobuf-java version that is compatible with KPL 1.0.x -->
+ <!-- see https://github.com/awslabs/amazon-kinesis-producer/blob/v1.0.4/java/amazon-kinesis-producer/pom.xml#L20 -->
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-bom</artifactId>
+ <version>${protobuf-java.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>software.amazon.kinesis</groupId>
+ <artifactId>amazon-kinesis-producer</artifactId>
+ <version>${amazon-kinesis-producer.version}</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>attach-shade-jar</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>${project.build.directory}/${project.build.finalName}.jar</file>
+ <type>jar</type>
+ <classifier>shaded</classifier>
+ </artifact>
+ </artifacts>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+ <minimizeJar>false</minimizeJar>
+ <artifactSet>
+ <includes>
+ <include>software.amazon.kinesis:amazon-kinesis-producer</include>
+ <include>com.google.protobuf:protobuf-java</include>
+ </includes>
+ <excludes>
+ <!-- This is required for execute shade multiple times without clean -->
+ <exclude>org.apache.pulsar:pulsar-io-kinesis-kpl-shaded</exclude>
+ </excludes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.google.protobuf</pattern>
+ <shadedPattern>org.apache.pulsar.io.kinesis.shaded.com.google.protobuf</shadedPattern>
+ </relocation>
+ </relocations>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ <exclude>META-INF/maven/${project.groupId}/${project.artifactId}/pom.xml</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <manifestEntries>
+ <Multi-Release>true</Multi-Release>
+ </manifestEntries>
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.PluginXmlResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <resource>META-INF/maven/${project.groupId}/${project.artifactId}/pom.xml</resource>
+ <file>${project.basedir}/dependency-reduced-pom.xml</file>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml
index 35f09540091..3bef295a06f 100644
--- a/pulsar-io/kinesis/pom.xml
+++ b/pulsar-io/kinesis/pom.xml
@@ -19,7 +19,7 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.pulsar</groupId>
@@ -31,8 +31,7 @@
<name>Pulsar IO :: Kinesis</name>
<properties>
- <amazon-kinesis-client.version>2.2.8</amazon-kinesis-client.version>
- <amazon-kinesis-producer.version>0.15.12</amazon-kinesis-producer.version>
+ <amazon-kinesis-client.version>3.1.2</amazon-kinesis-client.version>
<json-flattener.version>0.13.0</json-flattener.version>
<flatbuffers-java.version>1.9.0</flatbuffers-java.version>
<jaxb-api.version>2.3.0</jaxb-api.version>
@@ -50,6 +49,12 @@
</dependencyManagement>
<dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>pulsar-io-kinesis-kpl-shaded</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>pulsar-io-common</artifactId>
@@ -123,7 +128,6 @@
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>utils</artifactId>
- <version>2.22.12</version>
</dependency>
<dependency>
@@ -132,12 +136,7 @@
<version>${amazon-kinesis-client.version}</version>
</dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>amazon-kinesis-producer</artifactId>
- <version>${amazon-kinesis-producer.version}</version>
- </dependency>
- <!-- /kinesis dependencies -->
+ <!-- /kinesis dependencies -->
<dependency>
<groupId>com.google.flatbuffers</groupId>
@@ -177,6 +176,11 @@
<plugin>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-maven-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>**/amazon-kinesis-producer-*.jar</exclude>
+ </excludes>
+ </configuration>
</plugin>
</plugins>
</build>
diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
index 76686fbd899..db23972905a 100644
--- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
+++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
@@ -22,11 +22,6 @@ import static com.google.common.util.concurrent.Futures.addCallback;
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.services.kinesis.producer.KinesisProducer;
-import com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration;
-import com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration.ThreadingModel;
-import com.amazonaws.services.kinesis.producer.UserRecordFailedException;
-import com.amazonaws.services.kinesis.producer.UserRecordResult;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -56,6 +51,12 @@ import org.apache.pulsar.io.core.annotations.IOType;
import org.apache.pulsar.io.kinesis.KinesisSinkConfig.MessageFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.kinesis.producer.KinesisProducer;
+import software.amazon.kinesis.producer.KinesisProducerConfiguration;
+import software.amazon.kinesis.producer.KinesisProducerConfiguration.ThreadingModel;
+import software.amazon.kinesis.producer.UserRecordFailedException;
+import software.amazon.kinesis.producer.UserRecordResult;
/**
* A Kinesis sink which can be configured by {@link KinesisSinkConfig}.
@@ -191,10 +192,10 @@ public class KinesisSink extends AbstractAwsConnector implements Sink<GenericObj
&& kinesisSinkConfig.getSkipCertificateValidation()) {
kinesisConfig.setVerifyCertificate(false);
}
- AWSCredentialsProvider credentialsProvider = createCredentialProvider(
+ AwsCredentialsProvider credentialsProvider = createCredentialProvider(
kinesisSinkConfig.getAwsCredentialPluginName(),
kinesisSinkConfig.getAwsCredentialPluginParam())
- .getCredentialProvider();
+ .getV2CredentialsProvider();
kinesisConfig.setCredentialsProvider(credentialsProvider);
kinesisConfig.setNativeExecutable(StringUtils.trimToEmpty(kinesisSinkConfig.getNativeExecutable()));
kinesisConfig.setAggregationEnabled(kinesisSinkConfig.isAggregationEnabled());
diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSinkConfig.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSinkConfig.java
index 9d316e54419..d4f34a05596 100644
--- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSinkConfig.java
+++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSinkConfig.java
@@ -20,7 +20,6 @@ package org.apache.pulsar.io.kinesis;
import static com.google.common.base.Preconditions.checkArgument;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
-import com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.io.Serializable;
@@ -31,6 +30,7 @@ import lombok.EqualsAndHashCode;
import org.apache.pulsar.io.common.IOConfigUtils;
import org.apache.pulsar.io.core.SinkContext;
import org.apache.pulsar.io.core.annotations.FieldDoc;
+import software.amazon.kinesis.producer.KinesisProducerConfiguration;
@Data
@EqualsAndHashCode(callSuper = true)
@@ -245,7 +245,7 @@ public class KinesisSinkConfig extends BaseKinesisConfig implements Serializable
@FieldDoc(
defaultValue = "",
- help = "Extra KinesisProducerConfiguration parameters. See https://javadoc.io/static/com.amazonaws/amazon-kinesis-producer/0.15.2/index.html?com/amazonaws/services/kinesis/producer/KinesisProducerConfiguration.html for all the available parameters."
+ help = "Extra KinesisProducerConfiguration parameters. See https://javadoc.io/static/software.amazon.kinesis/amazon-kinesis-producer/1.0.4/software/amazon/kinesis/producer/KinesisProducerConfiguration.html for all the available parameters."
+ "Parameters that are explicitly set take preference over extra config.")
private Map<String, String> extraKinesisProducerConfig = new HashMap<>();
diff --git a/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSinkConfigTest.java b/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSinkConfigTest.java
index 1c6beee96b8..1a81965d38e 100644
--- a/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSinkConfigTest.java
+++ b/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSinkConfigTest.java
@@ -20,13 +20,13 @@ package org.apache.pulsar.io.kinesis;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
-import com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.pulsar.io.core.SinkContext;
import org.mockito.Mockito;
import org.testng.annotations.Test;
+import software.amazon.kinesis.producer.KinesisProducerConfiguration;
public class KinesisSinkConfigTest {
diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml
index e4341d6a707..123d257596a 100644
--- a/pulsar-io/pom.xml
+++ b/pulsar-io/pom.xml
@@ -56,6 +56,7 @@
<module>kafka</module>
<module>rabbitmq</module>
<module>kinesis</module>
+ <module>kinesis-kpl-shaded</module>
<module>hdfs3</module>
<module>jdbc</module>
<module>data-generator</module>
@@ -93,6 +94,7 @@
<module>http</module>
<module>rabbitmq</module>
<module>kinesis</module>
+ <module>kinesis-kpl-shaded</module>
<module>hdfs3</module>
<module>jdbc</module>
<module>data-generator</module>
diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml
index ed822f1f072..0a5f26f9c7a 100644
--- a/tests/integration/pom.xml
+++ b/tests/integration/pom.xml
@@ -222,7 +222,7 @@
<dependency>
<groupId>software.amazon.kinesis</groupId>
<artifactId>amazon-kinesis-client</artifactId>
- <version>2.2.8</version>
+ <version>3.1.2</version>
<scope>test</scope>
</dependency>
<dependency>