You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2022/04/04 00:52:18 UTC
[pulsar] branch master updated: [Python] Added build for Arm64 on Linux and Python 3.10 (#15004)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 2054dbf67c6 [Python] Added build for Arm64 on Linux and Python 3.10 (#15004)
2054dbf67c6 is described below
commit 2054dbf67c608eb14193b3fe7c4287bed9287669
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Sun Apr 3 17:52:11 2022 -0700
[Python] Added build for Arm64 on Linux and Python 3.10 (#15004)
* [Python] Added build for Arm64 on Linux and Python 3.10
* Fixed docker-build-python3.9.sh
* Added arch in docker/pulsar/pom.xml
* Fixed spacing in docker/pulsar/pom.xml
* Fixed build-wheels.sh when taking args
* Removed debug statement
* Fixed variable expansion
* Pass the arch to the script inside docker
---
docker/pulsar/pom.xml | 2 +-
pulsar-client-cpp/docker-build-python3.9.sh | 8 ++-
.../docker/build-wheel-file-within-docker.sh | 4 +-
pulsar-client-cpp/docker/build-wheels.sh | 52 +++++++++------
pulsar-client-cpp/docker/create-images.sh | 27 +++++---
pulsar-client-cpp/docker/manylinux2014/Dockerfile | 77 +++++++---------------
pulsar-client-cpp/python/CMakeLists.txt | 11 +++-
7 files changed, 89 insertions(+), 92 deletions(-)
diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
index 6d61417f049..7d229a157eb 100644
--- a/docker/pulsar/pom.xml
+++ b/docker/pulsar/pom.xml
@@ -75,7 +75,7 @@
<executable>${project.basedir}/../../pulsar-client-cpp/docker/build-wheels.sh</executable>
<arguments>
<!-- build python 3.8 -->
- <argument>3.8 cp38-cp38</argument>
+ <argument>3.8 cp38-cp38 x86_64</argument>
</arguments>
</configuration>
</execution>
diff --git a/pulsar-client-cpp/docker-build-python3.9.sh b/pulsar-client-cpp/docker-build-python3.9.sh
index 15cc2fdb102..db5c9abd82a 100755
--- a/pulsar-client-cpp/docker-build-python3.9.sh
+++ b/pulsar-client-cpp/docker-build-python3.9.sh
@@ -29,16 +29,18 @@ cd $ROOT_DIR/pulsar-client-cpp
# Build manylinux2014 build image
PYTHON_VERSION="3.9"
PYTHON_SPEC="cp39-cp39"
-IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC
+ARCH="x86_64"
+IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC-$ARCH
docker build -t $IMAGE_NAME ./docker/manylinux2014 \
--build-arg PYTHON_VERSION=$PYTHON_VERSION \
- --build-arg PYTHON_SPEC=$PYTHON_SPEC
+ --build-arg PYTHON_SPEC=$PYTHON_SPEC \
+ --build-arg ARCH=$ARCH
# Build wheel file
BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-pulsar-build}"
-IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC
+IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC-$ARCH
VOLUME_OPTION=${VOLUME_OPTION:-"-v $ROOT_DIR:/pulsar"}
COMMAND="/pulsar/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh"
diff --git a/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh b/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh
index 5d7f2a8385f..87650f9c3a8 100755
--- a/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh
+++ b/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh
@@ -32,7 +32,7 @@ cmake . -DPYTHON_INCLUDE_DIR=/opt/python/$PYTHON_SPEC/include/python$PYTHON_VERS
-DBUILD_WIRESHARK=OFF
make clean
-make _pulsar -j3 VERBOSE=1
+make _pulsar -j3
cd python
python setup.py bdist_wheel
@@ -43,4 +43,4 @@ python setup.py bdist_wheel
# Audit wheel will make sure no external dependencies are needed for
# the shared library and that only symbols supported by most linux
# distributions are used.
-auditwheel repair dist/pulsar_client*-$PYTHON_SPEC-linux_x86_64.whl
+auditwheel repair dist/pulsar_client*-$PYTHON_SPEC-linux_${ARCH}.whl
diff --git a/pulsar-client-cpp/docker/build-wheels.sh b/pulsar-client-cpp/docker/build-wheels.sh
index a61f21516eb..d3549fd0188 100755
--- a/pulsar-client-cpp/docker/build-wheels.sh
+++ b/pulsar-client-cpp/docker/build-wheels.sh
@@ -28,32 +28,41 @@ ROOT_DIR=`cd $(dirname $0)/../..; pwd`
cd $ROOT_DIR
PYTHON_VERSIONS=(
- '2.7 cp27-cp27mu'
- '2.7 cp27-cp27m'
- '3.5 cp35-cp35m'
- '3.6 cp36-cp36m'
- '3.7 cp37-cp37m'
- '3.8 cp38-cp38'
- '3.9 cp39-cp39'
+ '2.7 cp27-cp27mu x86_64'
+ '2.7 cp27-cp27m x86_64'
+ '3.5 cp35-cp35m x86_64'
+ '3.6 cp36-cp36m x86_64'
+ '3.7 cp37-cp37m x86_64'
+ '3.8 cp38-cp38 x86_64'
+ '3.9 cp39-cp39 x86_64'
+ '3.10 cp310-cp310 x86_64'
+ '3.7 cp37-cp37m aarch64'
+ '3.8 cp38-cp38 aarch64'
+ '3.9 cp39-cp39 aarch64'
+ '3.10 cp310-cp310 aarch64'
)
-function contains() {
- local n=$#
- local value=${!n}
- for ((i=1;i < $#;i++)) {
- if [ "${!i}" == "${value}" ]; then
- echo "y"
- return 0
+function contains_build_version {
+ for line in "${PYTHON_VERSIONS[@]}"; do
+ read -r -a v <<< "$line"
+ value="${v[0]} ${v[1]} ${v[2]}"
+
+ if [ "${build_version}" == "${value}" ]; then
+ # found
+ res=1
+ return
fi
- }
- echo "n"
- return 1
+ done
+
+ # not found
+ res=0
}
if [ $# -ge 1 ]; then
build_version=$@
- if [ $(contains "${PYTHON_VERSIONS[@]}" "${build_version}") == "y" ]; then
+ contains_build_version
+ if [ $res == 1 ]; then
PYTHON_VERSIONS=(
"${build_version}"
)
@@ -69,9 +78,10 @@ for line in "${PYTHON_VERSIONS[@]}"; do
read -r -a PY <<< "$line"
PYTHON_VERSION=${PY[0]}
PYTHON_SPEC=${PY[1]}
- echo "--------- Build Python wheel for $PYTHON_VERSION -- $PYTHON_SPEC"
+ ARCH=${PY[2]}
+ echo "--------- Build Python wheel for $PYTHON_VERSION -- $PYTHON_SPEC -- $ARCH"
- IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC
+ IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC-$ARCH
echo "Using image: $IMAGE"
@@ -79,6 +89,6 @@ for line in "${PYTHON_VERSIONS[@]}"; do
COMMAND="/pulsar/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh"
DOCKER_CMD="docker run -i ${VOLUME_OPTION} -e USE_FULL_POM_NAME -e NAME_POSTFIX ${IMAGE}"
- $DOCKER_CMD bash -c "${COMMAND}"
+ $DOCKER_CMD bash -c "ARCH=$ARCH ${COMMAND}"
done
diff --git a/pulsar-client-cpp/docker/create-images.sh b/pulsar-client-cpp/docker/create-images.sh
index 6aa1d69ef44..0a282375f7d 100755
--- a/pulsar-client-cpp/docker/create-images.sh
+++ b/pulsar-client-cpp/docker/create-images.sh
@@ -24,13 +24,18 @@
set -e
PYTHON_VERSIONS=(
- '2.7 cp27-cp27mu manylinux1'
- '2.7 cp27-cp27m manylinux1'
- '3.5 cp35-cp35m manylinux2014'
- '3.6 cp36-cp36m manylinux2014'
- '3.7 cp37-cp37m manylinux2014'
- '3.8 cp38-cp38 manylinux2014'
- '3.9 cp39-cp39 manylinux2014'
+ '2.7 cp27-cp27mu manylinux1 x86_64'
+ '2.7 cp27-cp27m manylinux1 x86_64'
+ '3.5 cp35-cp35m manylinux2014 x86_64'
+ '3.6 cp36-cp36m manylinux2014 x86_64'
+ '3.7 cp37-cp37m manylinux2014 x86_64'
+ '3.8 cp38-cp38 manylinux2014 x86_64'
+ '3.9 cp39-cp39 manylinux2014 x86_64'
+ '3.10 cp310-cp310 manylinux2014 x86_64'
+ '3.7 cp37-cp37m manylinux2014 aarch64'
+ '3.8 cp38-cp38 manylinux2014 aarch64'
+ '3.9 cp39-cp39 manylinux2014 aarch64'
+ '3.10 cp310-cp310 manylinux2014 aarch64'
)
for line in "${PYTHON_VERSIONS[@]}"; do
@@ -38,13 +43,15 @@ for line in "${PYTHON_VERSIONS[@]}"; do
PYTHON_VERSION=${PY[0]}
PYTHON_SPEC=${PY[1]}
BASE_IMAGE=${PY[2]}
- echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC"
+ ARCH=${PY[3]}
+ echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC -- $ARCH"
- IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC
+ IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC-$ARCH
docker build -t $IMAGE_NAME $BASE_IMAGE \
--build-arg PYTHON_VERSION=$PYTHON_VERSION \
- --build-arg PYTHON_SPEC=$PYTHON_SPEC
+ --build-arg PYTHON_SPEC=$PYTHON_SPEC \
+ --build-arg ARCH=$ARCH
echo "==== Successfully built image $IMAGE_NAME"
done
diff --git a/pulsar-client-cpp/docker/manylinux2014/Dockerfile b/pulsar-client-cpp/docker/manylinux2014/Dockerfile
index 850c0b7a8d0..5bc1b0315ed 100644
--- a/pulsar-client-cpp/docker/manylinux2014/Dockerfile
+++ b/pulsar-client-cpp/docker/manylinux2014/Dockerfile
@@ -17,10 +17,8 @@
# under the License.
#
-
-FROM quay.io/pypa/manylinux2014_x86_64
-
-RUN yum install -y gtest-devel
+ARG ARCH
+FROM quay.io/pypa/manylinux2014_${ARCH}
ARG PYTHON_VERSION
ARG PYTHON_SPEC
@@ -28,6 +26,10 @@ ARG PYTHON_SPEC
ENV PYTHON_VERSION=${PYTHON_VERSION}
ENV PYTHON_SPEC=${PYTHON_SPEC}
+ARG ARCH
+ENV ARCH=${ARCH}
+
+
ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}"
RUN ln -s /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION}m /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION}
@@ -50,80 +52,47 @@ RUN curl -O -L https://zlib.net/zlib-1.2.12.tar.gz && \
tar xvfz zlib-1.2.12.tar.gz && \
cd zlib-1.2.12 && \
CFLAGS="-fPIC -O3" ./configure && \
- make && make install && \
+ make -j8 && make install && \
rm -rf /zlib-1.2.12.tar.gz /zlib-1.2.12
# Compile OpenSSL
-RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_0j.tar.gz && \
- tar xvfz OpenSSL_1_1_0j.tar.gz && \
- cd openssl-OpenSSL_1_1_0j/ && \
- ./Configure -fPIC --prefix=/usr/local/ssl/ no-shared linux-x86_64 && \
- make && make install && \
- rm -rf /OpenSSL_1_1_0j.tar.gz /openssl-OpenSSL_1_1_0j
+RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_1n.tar.gz && \
+ tar xvfz OpenSSL_1_1_1n.tar.gz && \
+ cd openssl-OpenSSL_1_1_1n/ && \
+ ./Configure -fPIC --prefix=/usr/local/ssl/ no-shared linux-${ARCH} && \
+ make -j8 && make install && \
+ rm -rf /OpenSSL_1_1_1n.tar.gz /openssl-OpenSSL_1_1_1n
# Download and compile boost
-RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.68.0/source/boost_1_68_0.tar.gz && \
- tar xvfz boost_1_68_0.tar.gz && \
- cd /boost_1_68_0 && \
+RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz && \
+ tar xvfz boost_1_78_0.tar.gz && \
+ cd /boost_1_78_0 && \
./bootstrap.sh --with-libraries=program_options,filesystem,regex,thread,system,python && \
- ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \
- rm -rf /boost_1_68_0.tar.gz /boost_1_68_0
+ ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install -j8 && \
+ rm -rf /boost_1_78_0.tar.gz /boost_1_78_0
# Download and copile protoubf
RUN curl -O -L https://github.com/google/protobuf/releases/download/v3.3.0/protobuf-cpp-3.3.0.tar.gz && \
tar xvfz protobuf-cpp-3.3.0.tar.gz && \
cd protobuf-3.3.0/ && \
CXXFLAGS=-fPIC ./configure && \
- make && make install && ldconfig && \
+ make -j8 && make install && ldconfig && \
rm -rf /protobuf-cpp-3.3.0.tar.gz /protobuf-3.3.0
-# Compile APR
-RUN curl -O -L http://archive.apache.org/dist/apr/apr-1.5.2.tar.gz && \
- tar xvfz apr-1.5.2.tar.gz && \
- cd apr-1.5.2 && \
- CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure && \
- make && make install && \
- rm -rf /apr-1.5.2.tar.gz /apr-1.5.2
-
-# Compile APR-Util
-RUN curl -O -L http://archive.apache.org/dist/apr/apr-util-1.5.4.tar.gz && \
- tar xvfz apr-util-1.5.4.tar.gz && \
- cd apr-util-1.5.4 && \
- CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure -with-apr=/usr/local/apr && \
- make && make install && \
- rm -rf /apr-util-1.5.4.tar.gz /apr-util-1.5.4
-
-# Libtool
-RUN curl -L -O https://ftp.gnu.org/gnu/libtool/libtool-2.4.6.tar.gz && \
- tar xvfz libtool-2.4.6.tar.gz && \
- cd libtool-2.4.6 && \
- ./configure && \
- make && make install && \
- rm -rf /libtool-2.4.6.tar.gz /libtool-2.4.6
-
-# Compile log4cxx
-RUN curl -O -L https://github.com/apache/logging-log4cxx/archive/v0.11.0.tar.gz && \
- tar xvfz v0.11.0.tar.gz && \
- cd logging-log4cxx-0.11.0 && \
- ./autogen.sh && \
- CXXFLAGS=-fPIC ./configure && \
- make && make install && \
- rm -rf /v0.11.0.tar.gz /logging-log4cxx-0.11.0
-
# Compile expat
RUN curl -O -L https://github.com/libexpat/libexpat/archive/R_2_2_0.tar.gz && \
tar xfvz R_2_2_0.tar.gz && \
cd libexpat-R_2_2_0/expat && \
./buildconf.sh && \
CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure && \
- make && make installlib && \
+ make -j8 && make installlib && \
rm -rf /R_2_2_0.tar.gz /libexpat-R_2_2_0
RUN curl -O -L https://github.com/Kitware/CMake/archive/v3.12.1.tar.gz && \
tar xvfz v3.12.1.tar.gz && \
cd CMake-3.12.1 && \
./configure && \
- make && make install && \
+ make -j8 && make install && \
rm -rf /v3.12.1.tar.gz /CMake-3.12.1
# Zstandard
@@ -139,7 +108,7 @@ RUN curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1
tar xvfz snappy-1.1.3.tar.gz && \
cd snappy-1.1.3 && \
CXXFLAGS="-fPIC -O3" ./configure && \
- make && make install && \
+ make -j8 && make install && \
rm -rf /snappy-1.1.3 /snappy-1.1.3.tar.gz
# LibCurl
@@ -147,7 +116,7 @@ RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_61_0/curl-
tar xvfz curl-7.61.0.tar.gz && \
cd curl-7.61.0 && \
CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ && \
- make && make install && \
+ make -j8 && make install && \
rm -rf /curl-7.61.0.tar.gz /curl-7.61.0
RUN pip install twine
diff --git a/pulsar-client-cpp/python/CMakeLists.txt b/pulsar-client-cpp/python/CMakeLists.txt
index 30631cd8a50..ee4a6b2b032 100644
--- a/pulsar-client-cpp/python/CMakeLists.txt
+++ b/pulsar-client-cpp/python/CMakeLists.txt
@@ -60,6 +60,10 @@ if (NOT DEFINED ${Boost_PYTHON39-MT_LIBRARY})
set(Boost_PYTHON39-MT_LIBRARY ${Boost_PYTHON39_LIBRARY})
endif()
+if (NOT DEFINED ${Boost_PYTHON310-MT_LIBRARY})
+ set(Boost_PYTHON310-MT_LIBRARY ${Boost_PYTHON310_LIBRARY})
+endif()
+
# Try all possible boost-python variable namings
set(PYTHON_WRAPPER_LIBS ${Boost_PYTHON_LIBRARY}
${Boost_PYTHON3_LIBRARY}
@@ -69,7 +73,9 @@ set(PYTHON_WRAPPER_LIBS ${Boost_PYTHON_LIBRARY}
${Boost_PYTHON35_LIBRARY}
${Boost_PYTHON36_LIBRARY}
${Boost_PYTHON38_LIBRARY}
- ${Boost_PYTHON39_LIBRARY})
+ ${Boost_PYTHON39_LIBRARY}
+ ${Boost_PYTHON310_LIBRARY}
+ )
if (APPLE)
if (Boost_PYTHON27-MT_LIBRARY_RELEASE)
@@ -84,6 +90,9 @@ if (APPLE)
if (Boost_PYTHON39-MT_LIBRARY_RELEASE)
set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} ${Boost_PYTHON39-MT_LIBRARY_RELEASE})
endif ()
+ if (Boost_PYTHON310-MT_LIBRARY_RELEASE)
+ set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} ${Boost_PYTHON310-MT_LIBRARY_RELEASE})
+ endif ()
endif()
message(STATUS "Using Boost Python libs: ${PYTHON_WRAPPER_LIBS}")