You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2022/04/04 17:06:21 UTC
[pulsar] branch master updated: [Python] Added build for wheels on Alpine linux (#15016)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 72c20bea90c [Python] Added build for wheels on Alpine linux (#15016)
72c20bea90c is described below
commit 72c20bea90ccbcee9a103b64d02cf5ba9ebd2090
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Mon Apr 4 10:06:06 2022 -0700
[Python] Added build for wheels on Alpine linux (#15016)
---
docker/pulsar/pom.xml | 2 +-
pulsar-client-cpp/docker/build-wheels.sh | 24 ++---
pulsar-client-cpp/docker/create-images.sh | 17 +--
pulsar-client-cpp/docker/manylinux_musl/Dockerfile | 116 +++++++++++++++++++++
pulsar-client-cpp/docker/push-images.sh | 18 ++--
.../{create-images.sh => python-versions.sh} | 33 ++----
6 files changed, 141 insertions(+), 69 deletions(-)
diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
index 7d229a157eb..622fa5c64b8 100644
--- a/docker/pulsar/pom.xml
+++ b/docker/pulsar/pom.xml
@@ -75,7 +75,7 @@
<executable>${project.basedir}/../../pulsar-client-cpp/docker/build-wheels.sh</executable>
<arguments>
<!-- build python 3.8 -->
- <argument>3.8 cp38-cp38 x86_64</argument>
+ <argument>3.8 cp38-cp38 manylinux2014 x86_64</argument>
</arguments>
</configuration>
</execution>
diff --git a/pulsar-client-cpp/docker/build-wheels.sh b/pulsar-client-cpp/docker/build-wheels.sh
index d3549fd0188..25ac64f4846 100755
--- a/pulsar-client-cpp/docker/build-wheels.sh
+++ b/pulsar-client-cpp/docker/build-wheels.sh
@@ -27,25 +27,12 @@ BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}"
ROOT_DIR=`cd $(dirname $0)/../..; pwd`
cd $ROOT_DIR
-PYTHON_VERSIONS=(
- '2.7 cp27-cp27mu x86_64'
- '2.7 cp27-cp27m x86_64'
- '3.5 cp35-cp35m x86_64'
- '3.6 cp36-cp36m x86_64'
- '3.7 cp37-cp37m x86_64'
- '3.8 cp38-cp38 x86_64'
- '3.9 cp39-cp39 x86_64'
- '3.10 cp310-cp310 x86_64'
- '3.7 cp37-cp37m aarch64'
- '3.8 cp38-cp38 aarch64'
- '3.9 cp39-cp39 aarch64'
- '3.10 cp310-cp310 aarch64'
-)
+source ./pulsar-client-cpp/docker/python-versions.sh
function contains_build_version {
for line in "${PYTHON_VERSIONS[@]}"; do
read -r -a v <<< "$line"
- value="${v[0]} ${v[1]} ${v[2]}"
+ value="${v[0]} ${v[1]} ${v[2]} ${v[3]}"
if [ "${build_version}" == "${value}" ]; then
# found
@@ -78,10 +65,11 @@ for line in "${PYTHON_VERSIONS[@]}"; do
read -r -a PY <<< "$line"
PYTHON_VERSION=${PY[0]}
PYTHON_SPEC=${PY[1]}
- ARCH=${PY[2]}
- echo "--------- Build Python wheel for $PYTHON_VERSION -- $PYTHON_SPEC -- $ARCH"
+ IMAGE=${PY[2]}
+ ARCH=${PY[3]}
+ echo "--------- Build Python wheel for $PYTHON_VERSION -- $IMAGE -- $PYTHON_SPEC -- $ARCH"
- IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC-$ARCH
+ IMAGE=$BUILD_IMAGE_NAME:${IMAGE}-$PYTHON_SPEC-$ARCH
echo "Using image: $IMAGE"
diff --git a/pulsar-client-cpp/docker/create-images.sh b/pulsar-client-cpp/docker/create-images.sh
index 0a282375f7d..14938a40c2a 100755
--- a/pulsar-client-cpp/docker/create-images.sh
+++ b/pulsar-client-cpp/docker/create-images.sh
@@ -23,20 +23,7 @@
set -e
-PYTHON_VERSIONS=(
- '2.7 cp27-cp27mu manylinux1 x86_64'
- '2.7 cp27-cp27m manylinux1 x86_64'
- '3.5 cp35-cp35m manylinux2014 x86_64'
- '3.6 cp36-cp36m manylinux2014 x86_64'
- '3.7 cp37-cp37m manylinux2014 x86_64'
- '3.8 cp38-cp38 manylinux2014 x86_64'
- '3.9 cp39-cp39 manylinux2014 x86_64'
- '3.10 cp310-cp310 manylinux2014 x86_64'
- '3.7 cp37-cp37m manylinux2014 aarch64'
- '3.8 cp38-cp38 manylinux2014 aarch64'
- '3.9 cp39-cp39 manylinux2014 aarch64'
- '3.10 cp310-cp310 manylinux2014 aarch64'
-)
+source python-versions.sh
for line in "${PYTHON_VERSIONS[@]}"; do
read -r -a PY <<< "$line"
@@ -46,7 +33,7 @@ for line in "${PYTHON_VERSIONS[@]}"; do
ARCH=${PY[3]}
echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC -- $ARCH"
- IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC-$ARCH
+ IMAGE_NAME=pulsar-build:$BASE_IMAGE-$PYTHON_SPEC-$ARCH
docker build -t $IMAGE_NAME $BASE_IMAGE \
--build-arg PYTHON_VERSION=$PYTHON_VERSION \
diff --git a/pulsar-client-cpp/docker/manylinux_musl/Dockerfile b/pulsar-client-cpp/docker/manylinux_musl/Dockerfile
new file mode 100644
index 00000000000..89dd2d10755
--- /dev/null
+++ b/pulsar-client-cpp/docker/manylinux_musl/Dockerfile
@@ -0,0 +1,116 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+ARG ARCH
+FROM quay.io/pypa/musllinux_1_1_${ARCH}
+
+ARG PYTHON_VERSION
+ARG PYTHON_SPEC
+
+ENV PYTHON_VERSION=${PYTHON_VERSION}
+ENV PYTHON_SPEC=${PYTHON_SPEC}
+
+ARG ARCH
+ENV ARCH=${ARCH}
+
+
+ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}"
+
+RUN ln -s /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION}m /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION}
+
+# Perl (required for building OpenSSL)
+RUN curl -O -L https://www.cpan.org/src/5.0/perl-5.10.0.tar.gz && \
+ tar xvfz perl-5.10.0.tar.gz && \
+ cd perl-5.10.0 && \
+ ./configure.gnu --prefix=/usr/local/ && \
+ make && make install && \
+ rm -rf /perl-5.10.0.tar.gz /perl-5.10.0
+
+####################################
+# These dependencies can be found in Ubuntu but they're not compiled with -fPIC,
+# so they cannot be statically linked into a shared library
+####################################
+
+# ZLib
+RUN curl -O -L https://zlib.net/zlib-1.2.12.tar.gz && \
+ tar xvfz zlib-1.2.12.tar.gz && \
+ cd zlib-1.2.12 && \
+ CFLAGS="-fPIC -O3" ./configure && \
+ make -j8 && make install && \
+ rm -rf /zlib-1.2.12.tar.gz /zlib-1.2.12
+
+# Compile OpenSSL
+RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_1n.tar.gz && \
+ tar xvfz OpenSSL_1_1_1n.tar.gz && \
+ cd openssl-OpenSSL_1_1_1n/ && \
+ ./Configure -fPIC --prefix=/usr/local/ssl/ no-shared linux-${ARCH} && \
+ make -j8 && make install && \
+ rm -rf /OpenSSL_1_1_1n.tar.gz /openssl-OpenSSL_1_1_1n
+
+# Download and compile boost
+RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz && \
+ tar xvfz boost_1_78_0.tar.gz && \
+ cd /boost_1_78_0 && \
+ ./bootstrap.sh --with-libraries=program_options,filesystem,regex,thread,system,python && \
+ ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install -j8 && \
+ rm -rf /boost_1_78_0.tar.gz /boost_1_78_0
+
+# Download and copile protoubf
+RUN curl -O -L https://github.com/google/protobuf/releases/download/v3.20.0/protobuf-cpp-3.20.0.tar.gz && \
+ tar xvfz protobuf-cpp-3.20.0.tar.gz && \
+ cd protobuf-3.20.0/ && \
+ CXXFLAGS=-fPIC ./configure && \
+ make -j8 && make install && \
+ rm -rf /protobuf-cpp-3.20.0.tar.gz /protobuf-3.20.0
+
+RUN apk add cmake
+
+# Zstandard
+RUN curl -O -L https://github.com/facebook/zstd/releases/download/v1.3.7/zstd-1.3.7.tar.gz && \
+ tar xvfz zstd-1.3.7.tar.gz && \
+ cd zstd-1.3.7 && \
+ CFLAGS="-fPIC -O3" make -j8 && \
+ make install && \
+ rm -rf /zstd-1.3.7 /zstd-1.3.7.tar.gz
+
+# Snappy
+RUN curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz && \
+ tar xvfz snappy-1.1.3.tar.gz && \
+ cd snappy-1.1.3 && \
+ CXXFLAGS="-fPIC -O3" ./configure && \
+ make -j8 && make install && \
+ rm -rf /snappy-1.1.3 /snappy-1.1.3.tar.gz
+
+# LibCurl
+RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_61_0/curl-7.61.0.tar.gz && \
+ tar xvfz curl-7.61.0.tar.gz && \
+ cd curl-7.61.0 && \
+ CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ && \
+ make -j8 && make install && \
+ rm -rf /curl-7.61.0.tar.gz /curl-7.61.0
+
+RUN pip install twine
+RUN pip install fastavro
+RUN pip install six
+RUN pip install enum34
+
+
+ENV PYTHON_INCLUDE_DIR /opt/python/${PYTHON_SPEC}/include
+ENV PYTHON_LIBRARIES /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION}
+ENV OPENSSL_ROOT_DIR /usr/local/ssl/
diff --git a/pulsar-client-cpp/docker/push-images.sh b/pulsar-client-cpp/docker/push-images.sh
index 0501670e526..a1806fa29df 100755
--- a/pulsar-client-cpp/docker/push-images.sh
+++ b/pulsar-client-cpp/docker/push-images.sh
@@ -23,24 +23,18 @@
set -e
-DOCKER_ORG=apachepulsar
+source python-versions.sh
-PYTHON_VERSIONS=(
- '2.7 cp27-cp27mu'
- '2.7 cp27-cp27m'
- '3.5 cp35-cp35m'
- '3.6 cp36-cp36m'
- '3.7 cp37-cp37m'
- '3.8 cp38-cp38'
- '3.9 cp39-cp39'
-)
+DOCKER_ORG=apachepulsar
for line in "${PYTHON_VERSIONS[@]}"; do
read -r -a PY <<< "$line"
PYTHON_VERSION=${PY[0]}
PYTHON_SPEC=${PY[1]}
-
- IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC
+ BASE_IMAGE=${PY[2]}
+ ARCH=${PY[3]}
+
+ IMAGE_NAME=pulsar-build:$BASE_IMAGE-$PYTHON_SPEC-$ARCH
FULL_NAME=$DOCKER_ORG/$IMAGE_NAME
echo "IMAGE_NAME: $IMAGE_NAME"
diff --git a/pulsar-client-cpp/docker/create-images.sh b/pulsar-client-cpp/docker/python-versions.sh
old mode 100755
new mode 100644
similarity index 67%
copy from pulsar-client-cpp/docker/create-images.sh
copy to pulsar-client-cpp/docker/python-versions.sh
index 0a282375f7d..a264ee179c0
--- a/pulsar-client-cpp/docker/create-images.sh
+++ b/pulsar-client-cpp/docker/python-versions.sh
@@ -1,4 +1,3 @@
-#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -18,11 +17,6 @@
# under the License.
#
-
-# Create all the Docker images for variations of Python versions
-
-set -e
-
PYTHON_VERSIONS=(
'2.7 cp27-cp27mu manylinux1 x86_64'
'2.7 cp27-cp27m manylinux1 x86_64'
@@ -36,22 +30,15 @@ PYTHON_VERSIONS=(
'3.8 cp38-cp38 manylinux2014 aarch64'
'3.9 cp39-cp39 manylinux2014 aarch64'
'3.10 cp310-cp310 manylinux2014 aarch64'
-)
-for line in "${PYTHON_VERSIONS[@]}"; do
- read -r -a PY <<< "$line"
- PYTHON_VERSION=${PY[0]}
- PYTHON_SPEC=${PY[1]}
- BASE_IMAGE=${PY[2]}
- ARCH=${PY[3]}
- echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC -- $ARCH"
+ # Alpine compatible wheels
+ '3.7 cp37-cp37m manylinux_musl aarch64'
+ '3.8 cp38-cp38 manylinux_musl aarch64'
+ '3.9 cp39-cp39 manylinux_musl aarch64'
+ '3.10 cp310-cp310 manylinux_musl aarch64'
- IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC-$ARCH
-
- docker build -t $IMAGE_NAME $BASE_IMAGE \
- --build-arg PYTHON_VERSION=$PYTHON_VERSION \
- --build-arg PYTHON_SPEC=$PYTHON_SPEC \
- --build-arg ARCH=$ARCH
-
- echo "==== Successfully built image $IMAGE_NAME"
-done
+ '3.7 cp37-cp37m manylinux_musl x86_64'
+ '3.8 cp38-cp38 manylinux_musl x86_64'
+ '3.9 cp39-cp39 manylinux_musl x86_64'
+ '3.10 cp310-cp310 manylinux_musl x86_64'
+)