You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2022/04/07 04:11:35 UTC
[pulsar] branch branch-2.10 updated: [Python] Generate universal2 wheel files for MacOS (#15054)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new 460f1842615 [Python] Generate universal2 wheel files for MacOS (#15054)
460f1842615 is described below
commit 460f18426155de0bb17f8bb8172e63870b3f0917
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Wed Apr 6 21:05:53 2022 -0700
[Python] Generate universal2 wheel files for MacOS (#15054)
---
pulsar-client-cpp/CMakeLists.txt | 2 +-
pulsar-client-cpp/lib/checksum/crc32c_sse42.cc | 11 ++---
pulsar-client-cpp/python/build-mac-wheels.sh | 61 +++++++++++++++++++-------
3 files changed, 53 insertions(+), 21 deletions(-)
diff --git a/pulsar-client-cpp/CMakeLists.txt b/pulsar-client-cpp/CMakeLists.txt
index c9517de1f62..4fd18082f6c 100644
--- a/pulsar-client-cpp/CMakeLists.txt
+++ b/pulsar-client-cpp/CMakeLists.txt
@@ -97,7 +97,7 @@ else() # GCC or Clang are mostly compatible:
add_compile_options(-Wall -Wformat-security -Wvla -Werror)
# Turn off certain warnings that are too much pain for too little gain:
add_compile_options(-Wno-sign-compare -Wno-deprecated-declarations -Wno-error=cpp)
- if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR APPLE)
add_compile_options(-msse4.2 -mpclmul)
endif()
# Options unique to Clang or GCC:
diff --git a/pulsar-client-cpp/lib/checksum/crc32c_sse42.cc b/pulsar-client-cpp/lib/checksum/crc32c_sse42.cc
index 511140b17e1..c5dba04ba8e 100644
--- a/pulsar-client-cpp/lib/checksum/crc32c_sse42.cc
+++ b/pulsar-client-cpp/lib/checksum/crc32c_sse42.cc
@@ -31,7 +31,8 @@
#include "lib/checksum/crc32c_sw.h"
#include "gf2.hpp"
-#if BOOST_ARCH_X86_64
+#if BOOST_ARCH_X86_64 && !defined(__arm64__)
+#define PULSAR_X86_64
#include <nmmintrin.h> // SSE4.2
#include <wmmintrin.h> // PCLMUL
#else
@@ -44,7 +45,7 @@
#ifdef _MSC_VER
#include <intrin.h>
-#elif BOOST_ARCH_X86_64
+#elif defined(PULSAR_X86_64)
#include <cpuid.h>
#endif
@@ -79,7 +80,7 @@ bool crc32c_initialize() {
__cpuid(CPUInfo, 1);
has_sse42 = (CPUInfo[2] & cpuid_ecx_sse42) != 0;
has_pclmulqdq = (CPUInfo[2] & cpuid_ecx_pclmulqdq) != 0;
-#elif BOOST_ARCH_X86_64
+#elif defined(PULSAR_X86_64)
const uint32_t cpuid_ecx_sse42 = (1 << 20);
const uint32_t cpuid_ecx_pclmulqdq = (1 << 1);
unsigned int eax, ebx, ecx, edx;
@@ -116,7 +117,7 @@ void chunk_config::make_shift_table(size_t bytes, uint32_t table[256]) {
for (unsigned int i = 0; i < 256; ++i) table[i] = (const bitvector<32>)mul(m, bitvector<32>(i));
}
-#if BOOST_ARCH_X86_64
+#ifdef PULSAR_X86_64
static uint32_t crc32c_chunk(uint32_t crc, const void *buf, const chunk_config &config) {
DEBUG_PRINTF3(" crc32c_chunk(crc = 0x%08x, buf = %p, config.words = " SIZE_T_FORMAT ")", crc, buf,
@@ -259,7 +260,7 @@ uint32_t crc32c(uint32_t init, const void *buf, size_t len, const chunk_config *
return crc;
}
-#else // ! BOOST_ARCH_X86_64
+#else // ! PULSAR_X86_64
uint32_t crc32c(uint32_t init, const void *buf, size_t len, const chunk_config *config) {
// SSE 4.2 extension for hw implementation are not present
diff --git a/pulsar-client-cpp/python/build-mac-wheels.sh b/pulsar-client-cpp/python/build-mac-wheels.sh
index 68547b70f6b..0236eb44f79 100755
--- a/pulsar-client-cpp/python/build-mac-wheels.sh
+++ b/pulsar-client-cpp/python/build-mac-wheels.sh
@@ -20,6 +20,11 @@
set -e
+ARCHS=(
+ 'x86_64'
+ 'arm64'
+)
+
PYTHON_VERSIONS=(
'3.8 3.8.13'
'3.9 3.9.10'
@@ -45,10 +50,10 @@ cd "${ROOT_DIR}/pulsar-client-cpp"
CACHE_DIR=~/.pulsar-mac-wheels-cache
mkdir -p $CACHE_DIR
-PREFIX=$CACHE_DIR/install
-
cd $CACHE_DIR
+PREFIX=$CACHE_DIR/install
+
###############################################################################
for line in "${PYTHON_VERSIONS[@]}"; do
read -r -a PY <<< "$line"
@@ -63,7 +68,7 @@ for line in "${PYTHON_VERSIONS[@]}"; do
PY_PREFIX=$CACHE_DIR/py-$PYTHON_VERSION
pushd Python-${PYTHON_VERSION_LONG}
CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
- ./configure --prefix=$PY_PREFIX --enable-shared
+ ./configure --prefix=$PY_PREFIX --enable-shared --enable-universalsdk --with-universal-archs=universal2
make -j16
make install
@@ -83,7 +88,7 @@ if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then
curl -O -L https://zlib.net/zlib-${ZLIB_VERSION}.tar.gz
tar xvfz zlib-$ZLIB_VERSION.tar.gz
pushd zlib-$ZLIB_VERSION
- CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX
+ CFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX
make -j16
make install
touch .done
@@ -93,16 +98,35 @@ else
fi
###############################################################################
-if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION}/.done ]; then
+if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION}.done ]; then
echo "Building OpenSSL"
curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION}.tar.gz
+ # -arch arm64 -arch x86_64
tar xvfz OpenSSL_${OPENSSL_VERSION}.tar.gz
- pushd openssl-OpenSSL_${OPENSSL_VERSION}
- ./Configure -fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET} --prefix=$PREFIX no-shared darwin64-arm64-cc
+ mv openssl-OpenSSL_${OPENSSL_VERSION} openssl-OpenSSL_${OPENSSL_VERSION}-arm64
+ pushd openssl-OpenSSL_${OPENSSL_VERSION}-arm64
+ CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
+ ./Configure --prefix=$PREFIX no-shared darwin64-arm64-cc
make -j8
make install
- touch .done
popd
+
+ tar xvfz OpenSSL_${OPENSSL_VERSION}.tar.gz
+ mv openssl-OpenSSL_${OPENSSL_VERSION} openssl-OpenSSL_${OPENSSL_VERSION}-x86_64
+ pushd openssl-OpenSSL_${OPENSSL_VERSION}-x86_64
+ CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
+ ./Configure --prefix=$PREFIX no-shared darwin64-x86_64-cc
+ make -j8
+ make install
+ popd
+
+ # Create universal binaries
+ lipo -create openssl-OpenSSL_${OPENSSL_VERSION}-arm64/libssl.a openssl-OpenSSL_${OPENSSL_VERSION}-x86_64/libssl.a \
+ -output $PREFIX/lib/libssl.a
+ lipo -create openssl-OpenSSL_${OPENSSL_VERSION}-arm64/libcrypto.a openssl-OpenSSL_${OPENSSL_VERSION}-x86_64/libcrypto.a \
+ -output $PREFIX/lib/libcrypto.a
+
+ touch openssl-OpenSSL_${OPENSSL_VERSION}.done
else
echo "Using cached OpenSSL"
fi
@@ -133,7 +157,8 @@ for line in "${PYTHON_VERSIONS[@]}"; do
EOF
./bootstrap.sh --with-libraries=python --with-python=python3 --with-python-root=$PY_PREFIX \
--prefix=$CACHE_DIR/boost-py-$PYTHON_VERSION
- ./b2 address-model=64 cxxflags="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" link=static threading=multi \
+ ./b2 address-model=64 cxxflags="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
+ link=static threading=multi \
--user-config=./user-config.jam \
variant=release python=${PYTHON_VERSION} \
-j16 \
@@ -154,7 +179,8 @@ if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then
curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
tar xvfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
pushd protobuf-${PROTOBUF_VERSION}
- CXXFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX
+ CXXFLAGS="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
+ ./configure --prefix=$PREFIX
make -j16
make install
touch .done
@@ -169,7 +195,8 @@ if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then
curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz
tar xvfz zstd-${ZSTD_VERSION}.tar.gz
pushd zstd-${ZSTD_VERSION}
- CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX make -j16 install
+ CFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX \
+ make -j16 install
touch .done
popd
else
@@ -182,7 +209,8 @@ if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then
curl -O -L https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz
tar xvfz snappy-${SNAPPY_VERSION}.tar.gz
pushd snappy-${SNAPPY_VERSION}
- CXXFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX
+ CXXFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
+ ./configure --prefix=$PREFIX
make -j16
make install
touch .done
@@ -198,8 +226,10 @@ if [ ! -f curl-${CURL_VERSION}/.done ]; then
curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_}/curl-${CURL_VERSION}.tar.gz
tar xfz curl-${CURL_VERSION}.tar.gz
pushd curl-${CURL_VERSION}
- CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --with-ssl=$PREFIX \
- --without-nghttp2 --without-libidn2 --prefix=$PREFIX
+ CFLAGS="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
+ ./configure --with-ssl=$PREFIX \
+ --without-nghttp2 --without-libidn2 --disable-ldap \
+ --prefix=$PREFIX
make -j16 install
touch .done
popd
@@ -230,6 +260,7 @@ for line in "${PYTHON_VERSIONS[@]}"; do
PY_EXE=$PY_PREFIX/bin/python3
cmake . \
+ -DCMAKE_OSX_ARCHITECTURES='arm64;x86_64' \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \
-DCMAKE_OSX_SYSROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX${MACOSX_DEPLOYMENT_TARGET_MAJOR}.sdk \
-DCMAKE_INSTALL_PREFIX=$PREFIX \
@@ -252,4 +283,4 @@ for line in "${PYTHON_VERSIONS[@]}"; do
cd python
$PY_EXE setup.py bdist_wheel
-done
\ No newline at end of file
+done