You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2022/04/04 17:20:32 UTC
[pulsar] branch branch-2.10 updated (6898fc84312 -> 61281f11a80)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a change to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/pulsar.git
from 6898fc84312 [C++] Fix flaky tests about reference count (#14854)
new 99d86864529 [Python] Added build for Arm64 on Linux and Python 3.10 (#15004)
new 61281f11a80 [Python] Added build for wheels on Alpine linux (#15016)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docker/pulsar/pom.xml | 2 +-
pulsar-client-cpp/docker-build-python3.9.sh | 8 +-
.../docker/build-wheel-file-within-docker.sh | 4 +-
pulsar-client-cpp/docker/build-wheels.sh | 46 +++++----
pulsar-client-cpp/docker/create-images.sh | 18 ++--
pulsar-client-cpp/docker/manylinux2014/Dockerfile | 85 ++++++-----------
.../{manylinux1 => manylinux_musl}/Dockerfile | 105 ++++++---------------
pulsar-client-cpp/docker/push-images.sh | 18 ++--
pulsar-client-cpp/docker/python-versions.sh | 44 +++++++++
pulsar-client-cpp/python/CMakeLists.txt | 11 ++-
10 files changed, 153 insertions(+), 188 deletions(-)
copy pulsar-client-cpp/docker/{manylinux1 => manylinux_musl}/Dockerfile (54%)
create mode 100644 pulsar-client-cpp/docker/python-versions.sh
[pulsar] 02/02: [Python] Added build for wheels on Alpine linux (#15016)
Posted by mm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 61281f11a80e77c64f48453ba8d8aba486019c5c
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Mon Apr 4 10:06:06 2022 -0700
[Python] Added build for wheels on Alpine linux (#15016)
---
docker/pulsar/pom.xml | 2 +-
pulsar-client-cpp/docker/build-wheels.sh | 24 ++---
pulsar-client-cpp/docker/create-images.sh | 17 +--
pulsar-client-cpp/docker/manylinux_musl/Dockerfile | 116 +++++++++++++++++++++
pulsar-client-cpp/docker/push-images.sh | 18 ++--
.../{create-images.sh => python-versions.sh} | 33 ++----
6 files changed, 141 insertions(+), 69 deletions(-)
diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
index 5f633465426..b0facabacff 100644
--- a/docker/pulsar/pom.xml
+++ b/docker/pulsar/pom.xml
@@ -75,7 +75,7 @@
<executable>${project.basedir}/../../pulsar-client-cpp/docker/build-wheels.sh</executable>
<arguments>
<!-- build python 3.8 -->
- <argument>3.8 cp38-cp38 x86_64</argument>
+ <argument>3.8 cp38-cp38 manylinux2014 x86_64</argument>
</arguments>
</configuration>
</execution>
diff --git a/pulsar-client-cpp/docker/build-wheels.sh b/pulsar-client-cpp/docker/build-wheels.sh
index d3549fd0188..25ac64f4846 100755
--- a/pulsar-client-cpp/docker/build-wheels.sh
+++ b/pulsar-client-cpp/docker/build-wheels.sh
@@ -27,25 +27,12 @@ BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}"
ROOT_DIR=`cd $(dirname $0)/../..; pwd`
cd $ROOT_DIR
-PYTHON_VERSIONS=(
- '2.7 cp27-cp27mu x86_64'
- '2.7 cp27-cp27m x86_64'
- '3.5 cp35-cp35m x86_64'
- '3.6 cp36-cp36m x86_64'
- '3.7 cp37-cp37m x86_64'
- '3.8 cp38-cp38 x86_64'
- '3.9 cp39-cp39 x86_64'
- '3.10 cp310-cp310 x86_64'
- '3.7 cp37-cp37m aarch64'
- '3.8 cp38-cp38 aarch64'
- '3.9 cp39-cp39 aarch64'
- '3.10 cp310-cp310 aarch64'
-)
+source ./pulsar-client-cpp/docker/python-versions.sh
function contains_build_version {
for line in "${PYTHON_VERSIONS[@]}"; do
read -r -a v <<< "$line"
- value="${v[0]} ${v[1]} ${v[2]}"
+ value="${v[0]} ${v[1]} ${v[2]} ${v[3]}"
if [ "${build_version}" == "${value}" ]; then
# found
@@ -78,10 +65,11 @@ for line in "${PYTHON_VERSIONS[@]}"; do
read -r -a PY <<< "$line"
PYTHON_VERSION=${PY[0]}
PYTHON_SPEC=${PY[1]}
- ARCH=${PY[2]}
- echo "--------- Build Python wheel for $PYTHON_VERSION -- $PYTHON_SPEC -- $ARCH"
+ IMAGE=${PY[2]}
+ ARCH=${PY[3]}
+ echo "--------- Build Python wheel for $PYTHON_VERSION -- $IMAGE -- $PYTHON_SPEC -- $ARCH"
- IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC-$ARCH
+ IMAGE=$BUILD_IMAGE_NAME:${IMAGE}-$PYTHON_SPEC-$ARCH
echo "Using image: $IMAGE"
diff --git a/pulsar-client-cpp/docker/create-images.sh b/pulsar-client-cpp/docker/create-images.sh
index 0a282375f7d..14938a40c2a 100755
--- a/pulsar-client-cpp/docker/create-images.sh
+++ b/pulsar-client-cpp/docker/create-images.sh
@@ -23,20 +23,7 @@
set -e
-PYTHON_VERSIONS=(
- '2.7 cp27-cp27mu manylinux1 x86_64'
- '2.7 cp27-cp27m manylinux1 x86_64'
- '3.5 cp35-cp35m manylinux2014 x86_64'
- '3.6 cp36-cp36m manylinux2014 x86_64'
- '3.7 cp37-cp37m manylinux2014 x86_64'
- '3.8 cp38-cp38 manylinux2014 x86_64'
- '3.9 cp39-cp39 manylinux2014 x86_64'
- '3.10 cp310-cp310 manylinux2014 x86_64'
- '3.7 cp37-cp37m manylinux2014 aarch64'
- '3.8 cp38-cp38 manylinux2014 aarch64'
- '3.9 cp39-cp39 manylinux2014 aarch64'
- '3.10 cp310-cp310 manylinux2014 aarch64'
-)
+source python-versions.sh
for line in "${PYTHON_VERSIONS[@]}"; do
read -r -a PY <<< "$line"
@@ -46,7 +33,7 @@ for line in "${PYTHON_VERSIONS[@]}"; do
ARCH=${PY[3]}
echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC -- $ARCH"
- IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC-$ARCH
+ IMAGE_NAME=pulsar-build:$BASE_IMAGE-$PYTHON_SPEC-$ARCH
docker build -t $IMAGE_NAME $BASE_IMAGE \
--build-arg PYTHON_VERSION=$PYTHON_VERSION \
diff --git a/pulsar-client-cpp/docker/manylinux_musl/Dockerfile b/pulsar-client-cpp/docker/manylinux_musl/Dockerfile
new file mode 100644
index 00000000000..89dd2d10755
--- /dev/null
+++ b/pulsar-client-cpp/docker/manylinux_musl/Dockerfile
@@ -0,0 +1,116 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+ARG ARCH
+FROM quay.io/pypa/musllinux_1_1_${ARCH}
+
+ARG PYTHON_VERSION
+ARG PYTHON_SPEC
+
+ENV PYTHON_VERSION=${PYTHON_VERSION}
+ENV PYTHON_SPEC=${PYTHON_SPEC}
+
+ARG ARCH
+ENV ARCH=${ARCH}
+
+
+ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}"
+
+RUN ln -s /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION}m /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION}
+
+# Perl (required for building OpenSSL)
+RUN curl -O -L https://www.cpan.org/src/5.0/perl-5.10.0.tar.gz && \
+ tar xvfz perl-5.10.0.tar.gz && \
+ cd perl-5.10.0 && \
+ ./configure.gnu --prefix=/usr/local/ && \
+ make && make install && \
+ rm -rf /perl-5.10.0.tar.gz /perl-5.10.0
+
+####################################
+# These dependencies can be found in Ubuntu but they're not compiled with -fPIC,
+# so they cannot be statically linked into a shared library
+####################################
+
+# ZLib
+RUN curl -O -L https://zlib.net/zlib-1.2.12.tar.gz && \
+ tar xvfz zlib-1.2.12.tar.gz && \
+ cd zlib-1.2.12 && \
+ CFLAGS="-fPIC -O3" ./configure && \
+ make -j8 && make install && \
+ rm -rf /zlib-1.2.12.tar.gz /zlib-1.2.12
+
+# Compile OpenSSL
+RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_1n.tar.gz && \
+ tar xvfz OpenSSL_1_1_1n.tar.gz && \
+ cd openssl-OpenSSL_1_1_1n/ && \
+ ./Configure -fPIC --prefix=/usr/local/ssl/ no-shared linux-${ARCH} && \
+ make -j8 && make install && \
+ rm -rf /OpenSSL_1_1_1n.tar.gz /openssl-OpenSSL_1_1_1n
+
+# Download and compile boost
+RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz && \
+ tar xvfz boost_1_78_0.tar.gz && \
+ cd /boost_1_78_0 && \
+ ./bootstrap.sh --with-libraries=program_options,filesystem,regex,thread,system,python && \
+ ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install -j8 && \
+ rm -rf /boost_1_78_0.tar.gz /boost_1_78_0
+
+# Download and copile protoubf
+RUN curl -O -L https://github.com/google/protobuf/releases/download/v3.20.0/protobuf-cpp-3.20.0.tar.gz && \
+ tar xvfz protobuf-cpp-3.20.0.tar.gz && \
+ cd protobuf-3.20.0/ && \
+ CXXFLAGS=-fPIC ./configure && \
+ make -j8 && make install && \
+ rm -rf /protobuf-cpp-3.20.0.tar.gz /protobuf-3.20.0
+
+RUN apk add cmake
+
+# Zstandard
+RUN curl -O -L https://github.com/facebook/zstd/releases/download/v1.3.7/zstd-1.3.7.tar.gz && \
+ tar xvfz zstd-1.3.7.tar.gz && \
+ cd zstd-1.3.7 && \
+ CFLAGS="-fPIC -O3" make -j8 && \
+ make install && \
+ rm -rf /zstd-1.3.7 /zstd-1.3.7.tar.gz
+
+# Snappy
+RUN curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz && \
+ tar xvfz snappy-1.1.3.tar.gz && \
+ cd snappy-1.1.3 && \
+ CXXFLAGS="-fPIC -O3" ./configure && \
+ make -j8 && make install && \
+ rm -rf /snappy-1.1.3 /snappy-1.1.3.tar.gz
+
+# LibCurl
+RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_61_0/curl-7.61.0.tar.gz && \
+ tar xvfz curl-7.61.0.tar.gz && \
+ cd curl-7.61.0 && \
+ CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ && \
+ make -j8 && make install && \
+ rm -rf /curl-7.61.0.tar.gz /curl-7.61.0
+
+RUN pip install twine
+RUN pip install fastavro
+RUN pip install six
+RUN pip install enum34
+
+
+ENV PYTHON_INCLUDE_DIR /opt/python/${PYTHON_SPEC}/include
+ENV PYTHON_LIBRARIES /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION}
+ENV OPENSSL_ROOT_DIR /usr/local/ssl/
diff --git a/pulsar-client-cpp/docker/push-images.sh b/pulsar-client-cpp/docker/push-images.sh
index 0501670e526..a1806fa29df 100755
--- a/pulsar-client-cpp/docker/push-images.sh
+++ b/pulsar-client-cpp/docker/push-images.sh
@@ -23,24 +23,18 @@
set -e
-DOCKER_ORG=apachepulsar
+source python-versions.sh
-PYTHON_VERSIONS=(
- '2.7 cp27-cp27mu'
- '2.7 cp27-cp27m'
- '3.5 cp35-cp35m'
- '3.6 cp36-cp36m'
- '3.7 cp37-cp37m'
- '3.8 cp38-cp38'
- '3.9 cp39-cp39'
-)
+DOCKER_ORG=apachepulsar
for line in "${PYTHON_VERSIONS[@]}"; do
read -r -a PY <<< "$line"
PYTHON_VERSION=${PY[0]}
PYTHON_SPEC=${PY[1]}
-
- IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC
+ BASE_IMAGE=${PY[2]}
+ ARCH=${PY[3]}
+
+ IMAGE_NAME=pulsar-build:$BASE_IMAGE-$PYTHON_SPEC-$ARCH
FULL_NAME=$DOCKER_ORG/$IMAGE_NAME
echo "IMAGE_NAME: $IMAGE_NAME"
diff --git a/pulsar-client-cpp/docker/create-images.sh b/pulsar-client-cpp/docker/python-versions.sh
old mode 100755
new mode 100644
similarity index 67%
copy from pulsar-client-cpp/docker/create-images.sh
copy to pulsar-client-cpp/docker/python-versions.sh
index 0a282375f7d..a264ee179c0
--- a/pulsar-client-cpp/docker/create-images.sh
+++ b/pulsar-client-cpp/docker/python-versions.sh
@@ -1,4 +1,3 @@
-#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -18,11 +17,6 @@
# under the License.
#
-
-# Create all the Docker images for variations of Python versions
-
-set -e
-
PYTHON_VERSIONS=(
'2.7 cp27-cp27mu manylinux1 x86_64'
'2.7 cp27-cp27m manylinux1 x86_64'
@@ -36,22 +30,15 @@ PYTHON_VERSIONS=(
'3.8 cp38-cp38 manylinux2014 aarch64'
'3.9 cp39-cp39 manylinux2014 aarch64'
'3.10 cp310-cp310 manylinux2014 aarch64'
-)
-for line in "${PYTHON_VERSIONS[@]}"; do
- read -r -a PY <<< "$line"
- PYTHON_VERSION=${PY[0]}
- PYTHON_SPEC=${PY[1]}
- BASE_IMAGE=${PY[2]}
- ARCH=${PY[3]}
- echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC -- $ARCH"
+ # Alpine compatible wheels
+ '3.7 cp37-cp37m manylinux_musl aarch64'
+ '3.8 cp38-cp38 manylinux_musl aarch64'
+ '3.9 cp39-cp39 manylinux_musl aarch64'
+ '3.10 cp310-cp310 manylinux_musl aarch64'
- IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC-$ARCH
-
- docker build -t $IMAGE_NAME $BASE_IMAGE \
- --build-arg PYTHON_VERSION=$PYTHON_VERSION \
- --build-arg PYTHON_SPEC=$PYTHON_SPEC \
- --build-arg ARCH=$ARCH
-
- echo "==== Successfully built image $IMAGE_NAME"
-done
+ '3.7 cp37-cp37m manylinux_musl x86_64'
+ '3.8 cp38-cp38 manylinux_musl x86_64'
+ '3.9 cp39-cp39 manylinux_musl x86_64'
+ '3.10 cp310-cp310 manylinux_musl x86_64'
+)
[pulsar] 01/02: [Python] Added build for Arm64 on Linux and Python 3.10 (#15004)
Posted by mm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 99d868645298f5debf15d3ab2d72927bcdbe4386
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Sun Apr 3 17:52:11 2022 -0700
[Python] Added build for Arm64 on Linux and Python 3.10 (#15004)
* [Python] Added build for Arm64 on Linux and Python 3.10
* Fixed docker-build-python3.9.sh
* Added arch in docker/pulsar/pom.xml
* Fixed spacing in docker/pulsar/pom.xml
* Fixed build-wheels.sh when taking args
* Removed debug statement
* Fixed variable expansion
* Pass the arch to the script inside docker
---
docker/pulsar/pom.xml | 2 +-
pulsar-client-cpp/docker-build-python3.9.sh | 8 +-
.../docker/build-wheel-file-within-docker.sh | 4 +-
pulsar-client-cpp/docker/build-wheels.sh | 52 +++++++------
pulsar-client-cpp/docker/create-images.sh | 27 ++++---
pulsar-client-cpp/docker/manylinux2014/Dockerfile | 85 +++++++---------------
pulsar-client-cpp/python/CMakeLists.txt | 11 ++-
7 files changed, 93 insertions(+), 96 deletions(-)
diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
index 908e6a7656b..5f633465426 100644
--- a/docker/pulsar/pom.xml
+++ b/docker/pulsar/pom.xml
@@ -75,7 +75,7 @@
<executable>${project.basedir}/../../pulsar-client-cpp/docker/build-wheels.sh</executable>
<arguments>
<!-- build python 3.8 -->
- <argument>3.8 cp38-cp38</argument>
+ <argument>3.8 cp38-cp38 x86_64</argument>
</arguments>
</configuration>
</execution>
diff --git a/pulsar-client-cpp/docker-build-python3.9.sh b/pulsar-client-cpp/docker-build-python3.9.sh
index 15cc2fdb102..db5c9abd82a 100755
--- a/pulsar-client-cpp/docker-build-python3.9.sh
+++ b/pulsar-client-cpp/docker-build-python3.9.sh
@@ -29,16 +29,18 @@ cd $ROOT_DIR/pulsar-client-cpp
# Build manylinux2014 build image
PYTHON_VERSION="3.9"
PYTHON_SPEC="cp39-cp39"
-IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC
+ARCH="x86_64"
+IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC-$ARCH
docker build -t $IMAGE_NAME ./docker/manylinux2014 \
--build-arg PYTHON_VERSION=$PYTHON_VERSION \
- --build-arg PYTHON_SPEC=$PYTHON_SPEC
+ --build-arg PYTHON_SPEC=$PYTHON_SPEC \
+ --build-arg ARCH=$ARCH
# Build wheel file
BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-pulsar-build}"
-IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC
+IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC-$ARCH
VOLUME_OPTION=${VOLUME_OPTION:-"-v $ROOT_DIR:/pulsar"}
COMMAND="/pulsar/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh"
diff --git a/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh b/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh
index 5d7f2a8385f..87650f9c3a8 100755
--- a/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh
+++ b/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh
@@ -32,7 +32,7 @@ cmake . -DPYTHON_INCLUDE_DIR=/opt/python/$PYTHON_SPEC/include/python$PYTHON_VERS
-DBUILD_WIRESHARK=OFF
make clean
-make _pulsar -j3 VERBOSE=1
+make _pulsar -j3
cd python
python setup.py bdist_wheel
@@ -43,4 +43,4 @@ python setup.py bdist_wheel
# Audit wheel will make sure no external dependencies are needed for
# the shared library and that only symbols supported by most linux
# distributions are used.
-auditwheel repair dist/pulsar_client*-$PYTHON_SPEC-linux_x86_64.whl
+auditwheel repair dist/pulsar_client*-$PYTHON_SPEC-linux_${ARCH}.whl
diff --git a/pulsar-client-cpp/docker/build-wheels.sh b/pulsar-client-cpp/docker/build-wheels.sh
index a61f21516eb..d3549fd0188 100755
--- a/pulsar-client-cpp/docker/build-wheels.sh
+++ b/pulsar-client-cpp/docker/build-wheels.sh
@@ -28,32 +28,41 @@ ROOT_DIR=`cd $(dirname $0)/../..; pwd`
cd $ROOT_DIR
PYTHON_VERSIONS=(
- '2.7 cp27-cp27mu'
- '2.7 cp27-cp27m'
- '3.5 cp35-cp35m'
- '3.6 cp36-cp36m'
- '3.7 cp37-cp37m'
- '3.8 cp38-cp38'
- '3.9 cp39-cp39'
+ '2.7 cp27-cp27mu x86_64'
+ '2.7 cp27-cp27m x86_64'
+ '3.5 cp35-cp35m x86_64'
+ '3.6 cp36-cp36m x86_64'
+ '3.7 cp37-cp37m x86_64'
+ '3.8 cp38-cp38 x86_64'
+ '3.9 cp39-cp39 x86_64'
+ '3.10 cp310-cp310 x86_64'
+ '3.7 cp37-cp37m aarch64'
+ '3.8 cp38-cp38 aarch64'
+ '3.9 cp39-cp39 aarch64'
+ '3.10 cp310-cp310 aarch64'
)
-function contains() {
- local n=$#
- local value=${!n}
- for ((i=1;i < $#;i++)) {
- if [ "${!i}" == "${value}" ]; then
- echo "y"
- return 0
+function contains_build_version {
+ for line in "${PYTHON_VERSIONS[@]}"; do
+ read -r -a v <<< "$line"
+ value="${v[0]} ${v[1]} ${v[2]}"
+
+ if [ "${build_version}" == "${value}" ]; then
+ # found
+ res=1
+ return
fi
- }
- echo "n"
- return 1
+ done
+
+ # not found
+ res=0
}
if [ $# -ge 1 ]; then
build_version=$@
- if [ $(contains "${PYTHON_VERSIONS[@]}" "${build_version}") == "y" ]; then
+ contains_build_version
+ if [ $res == 1 ]; then
PYTHON_VERSIONS=(
"${build_version}"
)
@@ -69,9 +78,10 @@ for line in "${PYTHON_VERSIONS[@]}"; do
read -r -a PY <<< "$line"
PYTHON_VERSION=${PY[0]}
PYTHON_SPEC=${PY[1]}
- echo "--------- Build Python wheel for $PYTHON_VERSION -- $PYTHON_SPEC"
+ ARCH=${PY[2]}
+ echo "--------- Build Python wheel for $PYTHON_VERSION -- $PYTHON_SPEC -- $ARCH"
- IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC
+ IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC-$ARCH
echo "Using image: $IMAGE"
@@ -79,6 +89,6 @@ for line in "${PYTHON_VERSIONS[@]}"; do
COMMAND="/pulsar/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh"
DOCKER_CMD="docker run -i ${VOLUME_OPTION} -e USE_FULL_POM_NAME -e NAME_POSTFIX ${IMAGE}"
- $DOCKER_CMD bash -c "${COMMAND}"
+ $DOCKER_CMD bash -c "ARCH=$ARCH ${COMMAND}"
done
diff --git a/pulsar-client-cpp/docker/create-images.sh b/pulsar-client-cpp/docker/create-images.sh
index 6aa1d69ef44..0a282375f7d 100755
--- a/pulsar-client-cpp/docker/create-images.sh
+++ b/pulsar-client-cpp/docker/create-images.sh
@@ -24,13 +24,18 @@
set -e
PYTHON_VERSIONS=(
- '2.7 cp27-cp27mu manylinux1'
- '2.7 cp27-cp27m manylinux1'
- '3.5 cp35-cp35m manylinux2014'
- '3.6 cp36-cp36m manylinux2014'
- '3.7 cp37-cp37m manylinux2014'
- '3.8 cp38-cp38 manylinux2014'
- '3.9 cp39-cp39 manylinux2014'
+ '2.7 cp27-cp27mu manylinux1 x86_64'
+ '2.7 cp27-cp27m manylinux1 x86_64'
+ '3.5 cp35-cp35m manylinux2014 x86_64'
+ '3.6 cp36-cp36m manylinux2014 x86_64'
+ '3.7 cp37-cp37m manylinux2014 x86_64'
+ '3.8 cp38-cp38 manylinux2014 x86_64'
+ '3.9 cp39-cp39 manylinux2014 x86_64'
+ '3.10 cp310-cp310 manylinux2014 x86_64'
+ '3.7 cp37-cp37m manylinux2014 aarch64'
+ '3.8 cp38-cp38 manylinux2014 aarch64'
+ '3.9 cp39-cp39 manylinux2014 aarch64'
+ '3.10 cp310-cp310 manylinux2014 aarch64'
)
for line in "${PYTHON_VERSIONS[@]}"; do
@@ -38,13 +43,15 @@ for line in "${PYTHON_VERSIONS[@]}"; do
PYTHON_VERSION=${PY[0]}
PYTHON_SPEC=${PY[1]}
BASE_IMAGE=${PY[2]}
- echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC"
+ ARCH=${PY[3]}
+ echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC -- $ARCH"
- IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC
+ IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC-$ARCH
docker build -t $IMAGE_NAME $BASE_IMAGE \
--build-arg PYTHON_VERSION=$PYTHON_VERSION \
- --build-arg PYTHON_SPEC=$PYTHON_SPEC
+ --build-arg PYTHON_SPEC=$PYTHON_SPEC \
+ --build-arg ARCH=$ARCH
echo "==== Successfully built image $IMAGE_NAME"
done
diff --git a/pulsar-client-cpp/docker/manylinux2014/Dockerfile b/pulsar-client-cpp/docker/manylinux2014/Dockerfile
index 20a7247e30a..5bc1b0315ed 100644
--- a/pulsar-client-cpp/docker/manylinux2014/Dockerfile
+++ b/pulsar-client-cpp/docker/manylinux2014/Dockerfile
@@ -17,10 +17,8 @@
# under the License.
#
-
-FROM quay.io/pypa/manylinux2014_x86_64
-
-RUN yum install -y gtest-devel
+ARG ARCH
+FROM quay.io/pypa/manylinux2014_${ARCH}
ARG PYTHON_VERSION
ARG PYTHON_SPEC
@@ -28,6 +26,10 @@ ARG PYTHON_SPEC
ENV PYTHON_VERSION=${PYTHON_VERSION}
ENV PYTHON_SPEC=${PYTHON_SPEC}
+ARG ARCH
+ENV ARCH=${ARCH}
+
+
ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}"
RUN ln -s /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION}m /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION}
@@ -46,84 +48,51 @@ RUN curl -O -L https://www.cpan.org/src/5.0/perl-5.10.0.tar.gz && \
####################################
# ZLib
-RUN curl -O -L https://zlib.net/zlib-1.2.11.tar.gz && \
- tar xvfz zlib-1.2.11.tar.gz && \
- cd zlib-1.2.11 && \
+RUN curl -O -L https://zlib.net/zlib-1.2.12.tar.gz && \
+ tar xvfz zlib-1.2.12.tar.gz && \
+ cd zlib-1.2.12 && \
CFLAGS="-fPIC -O3" ./configure && \
- make && make install && \
- rm -rf /zlib-1.2.11.tar.gz /zlib-1.2.11
+ make -j8 && make install && \
+ rm -rf /zlib-1.2.12.tar.gz /zlib-1.2.12
# Compile OpenSSL
-RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_0j.tar.gz && \
- tar xvfz OpenSSL_1_1_0j.tar.gz && \
- cd openssl-OpenSSL_1_1_0j/ && \
- ./Configure -fPIC --prefix=/usr/local/ssl/ no-shared linux-x86_64 && \
- make && make install && \
- rm -rf /OpenSSL_1_1_0j.tar.gz /openssl-OpenSSL_1_1_0j
+RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_1n.tar.gz && \
+ tar xvfz OpenSSL_1_1_1n.tar.gz && \
+ cd openssl-OpenSSL_1_1_1n/ && \
+ ./Configure -fPIC --prefix=/usr/local/ssl/ no-shared linux-${ARCH} && \
+ make -j8 && make install && \
+ rm -rf /OpenSSL_1_1_1n.tar.gz /openssl-OpenSSL_1_1_1n
# Download and compile boost
-RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.68.0/source/boost_1_68_0.tar.gz && \
- tar xvfz boost_1_68_0.tar.gz && \
- cd /boost_1_68_0 && \
+RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz && \
+ tar xvfz boost_1_78_0.tar.gz && \
+ cd /boost_1_78_0 && \
./bootstrap.sh --with-libraries=program_options,filesystem,regex,thread,system,python && \
- ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \
- rm -rf /boost_1_68_0.tar.gz /boost_1_68_0
+ ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install -j8 && \
+ rm -rf /boost_1_78_0.tar.gz /boost_1_78_0
# Download and copile protoubf
RUN curl -O -L https://github.com/google/protobuf/releases/download/v3.3.0/protobuf-cpp-3.3.0.tar.gz && \
tar xvfz protobuf-cpp-3.3.0.tar.gz && \
cd protobuf-3.3.0/ && \
CXXFLAGS=-fPIC ./configure && \
- make && make install && ldconfig && \
+ make -j8 && make install && ldconfig && \
rm -rf /protobuf-cpp-3.3.0.tar.gz /protobuf-3.3.0
-# Compile APR
-RUN curl -O -L http://archive.apache.org/dist/apr/apr-1.5.2.tar.gz && \
- tar xvfz apr-1.5.2.tar.gz && \
- cd apr-1.5.2 && \
- CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure && \
- make && make install && \
- rm -rf /apr-1.5.2.tar.gz /apr-1.5.2
-
-# Compile APR-Util
-RUN curl -O -L http://archive.apache.org/dist/apr/apr-util-1.5.4.tar.gz && \
- tar xvfz apr-util-1.5.4.tar.gz && \
- cd apr-util-1.5.4 && \
- CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure -with-apr=/usr/local/apr && \
- make && make install && \
- rm -rf /apr-util-1.5.4.tar.gz /apr-util-1.5.4
-
-# Libtool
-RUN curl -L -O https://ftp.gnu.org/gnu/libtool/libtool-2.4.6.tar.gz && \
- tar xvfz libtool-2.4.6.tar.gz && \
- cd libtool-2.4.6 && \
- ./configure && \
- make && make install && \
- rm -rf /libtool-2.4.6.tar.gz /libtool-2.4.6
-
-# Compile log4cxx
-RUN curl -O -L https://github.com/apache/logging-log4cxx/archive/v0.11.0.tar.gz && \
- tar xvfz v0.11.0.tar.gz && \
- cd logging-log4cxx-0.11.0 && \
- ./autogen.sh && \
- CXXFLAGS=-fPIC ./configure && \
- make && make install && \
- rm -rf /v0.11.0.tar.gz /logging-log4cxx-0.11.0
-
# Compile expat
RUN curl -O -L https://github.com/libexpat/libexpat/archive/R_2_2_0.tar.gz && \
tar xfvz R_2_2_0.tar.gz && \
cd libexpat-R_2_2_0/expat && \
./buildconf.sh && \
CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure && \
- make && make installlib && \
+ make -j8 && make installlib && \
rm -rf /R_2_2_0.tar.gz /libexpat-R_2_2_0
RUN curl -O -L https://github.com/Kitware/CMake/archive/v3.12.1.tar.gz && \
tar xvfz v3.12.1.tar.gz && \
cd CMake-3.12.1 && \
./configure && \
- make && make install && \
+ make -j8 && make install && \
rm -rf /v3.12.1.tar.gz /CMake-3.12.1
# Zstandard
@@ -139,7 +108,7 @@ RUN curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1
tar xvfz snappy-1.1.3.tar.gz && \
cd snappy-1.1.3 && \
CXXFLAGS="-fPIC -O3" ./configure && \
- make && make install && \
+ make -j8 && make install && \
rm -rf /snappy-1.1.3 /snappy-1.1.3.tar.gz
# LibCurl
@@ -147,7 +116,7 @@ RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_61_0/curl-
tar xvfz curl-7.61.0.tar.gz && \
cd curl-7.61.0 && \
CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ && \
- make && make install && \
+ make -j8 && make install && \
rm -rf /curl-7.61.0.tar.gz /curl-7.61.0
RUN pip install twine
diff --git a/pulsar-client-cpp/python/CMakeLists.txt b/pulsar-client-cpp/python/CMakeLists.txt
index 30631cd8a50..ee4a6b2b032 100644
--- a/pulsar-client-cpp/python/CMakeLists.txt
+++ b/pulsar-client-cpp/python/CMakeLists.txt
@@ -60,6 +60,10 @@ if (NOT DEFINED ${Boost_PYTHON39-MT_LIBRARY})
set(Boost_PYTHON39-MT_LIBRARY ${Boost_PYTHON39_LIBRARY})
endif()
+if (NOT DEFINED ${Boost_PYTHON310-MT_LIBRARY})
+ set(Boost_PYTHON310-MT_LIBRARY ${Boost_PYTHON310_LIBRARY})
+endif()
+
# Try all possible boost-python variable namings
set(PYTHON_WRAPPER_LIBS ${Boost_PYTHON_LIBRARY}
${Boost_PYTHON3_LIBRARY}
@@ -69,7 +73,9 @@ set(PYTHON_WRAPPER_LIBS ${Boost_PYTHON_LIBRARY}
${Boost_PYTHON35_LIBRARY}
${Boost_PYTHON36_LIBRARY}
${Boost_PYTHON38_LIBRARY}
- ${Boost_PYTHON39_LIBRARY})
+ ${Boost_PYTHON39_LIBRARY}
+ ${Boost_PYTHON310_LIBRARY}
+ )
if (APPLE)
if (Boost_PYTHON27-MT_LIBRARY_RELEASE)
@@ -84,6 +90,9 @@ if (APPLE)
if (Boost_PYTHON39-MT_LIBRARY_RELEASE)
set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} ${Boost_PYTHON39-MT_LIBRARY_RELEASE})
endif ()
+ if (Boost_PYTHON310-MT_LIBRARY_RELEASE)
+ set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} ${Boost_PYTHON310-MT_LIBRARY_RELEASE})
+ endif ()
endif()
message(STATUS "Using Boost Python libs: ${PYTHON_WRAPPER_LIBS}")